{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use Soft Actor-Critic with Auto $\\alpha$ Tuning to Play LunarLander-v2\n",
    "\n",
    "PyTorch version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "import copy\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "torch.manual_seed(0)\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.distributions as distributions\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "08:49:53 [INFO] env: <LunarLander<LunarLander-v2>>\n",
      "08:49:53 [INFO] action_space: Discrete(4)\n",
      "08:49:53 [INFO] observation_space: Box(-inf, inf, (8,), float32)\n",
      "08:49:53 [INFO] reward_range: (-inf, inf)\n",
      "08:49:53 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}\n",
      "08:49:53 [INFO] _max_episode_steps: 1000\n",
      "08:49:53 [INFO] _elapsed_steps: None\n",
      "08:49:53 [INFO] id: LunarLander-v2\n",
      "08:49:53 [INFO] entry_point: gym.envs.box2d:LunarLander\n",
      "08:49:53 [INFO] reward_threshold: 200\n",
      "08:49:53 [INFO] nondeterministic: False\n",
      "08:49:53 [INFO] max_episode_steps: 1000\n",
      "08:49:53 [INFO] _kwargs: {}\n",
      "08:49:53 [INFO] _env_name: LunarLander\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('LunarLander-v2')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNReplayer:\n",
    "    def __init__(self, capacity):\n",
    "        self.memory = pd.DataFrame(index=range(capacity),\n",
    "                columns=['state', 'action', 'reward', 'next_state', 'done'])\n",
    "        self.i = 0\n",
    "        self.count = 0\n",
    "        self.capacity = capacity\n",
    "\n",
    "    def store(self, *args):\n",
    "        self.memory.loc[self.i] = args\n",
    "        self.i = (self.i + 1) % self.capacity\n",
    "        self.count = min(self.count + 1, self.capacity)\n",
    "\n",
    "    def sample(self, size):\n",
    "        indices = np.random.choice(self.count, size=size)\n",
    "        return (np.stack(self.memory.loc[indices, field]) for field in\n",
    "                self.memory.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SACAgent:\n",
    "    def __init__(self, env):\n",
    "        state_dim = env.observation_space.shape[0]\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.replayer = DQNReplayer(10000)\n",
    "\n",
    "        # create alpha\n",
    "        self.target_entropy = np.log(self.action_n) / 4.\n",
    "        self.ln_alpha_tensor = torch.zeros(1, requires_grad=True)\n",
    "        self.alpha_optimizer = optim.Adam([self.ln_alpha_tensor,], lr=3e-4)\n",
    "\n",
    "        # create actor\n",
    "        self.actor_net = self.build_net(input_size=state_dim,\n",
    "                hidden_sizes=[256, 256],\n",
    "                output_size=self.action_n, output_activator=nn.Softmax(-1))\n",
    "        self.actor_optimizer = optim.Adam(self.actor_net.parameters(), lr=3e-4)\n",
    "\n",
    "        # create V critic\n",
    "        self.v_evaluate_net = self.build_net(input_size=state_dim,\n",
    "                hidden_sizes=[256, 256])\n",
    "        self.v_target_net = copy.deepcopy(self.v_evaluate_net)\n",
    "        self.v_optimizer = optim.Adam(self.v_evaluate_net.parameters(), lr=3e-4)\n",
    "        self.v_loss = nn.MSELoss()\n",
    "\n",
    "        # create Q critic\n",
    "        self.q0_net = self.build_net(input_size=state_dim,\n",
    "                hidden_sizes=[256, 256], output_size=self.action_n)\n",
    "        self.q1_net = self.build_net(input_size=state_dim,\n",
    "                hidden_sizes=[256, 256], output_size=self.action_n)\n",
    "        self.q0_loss = nn.MSELoss()\n",
    "        self.q1_loss = nn.MSELoss()\n",
    "        self.q0_optimizer = optim.Adam(self.q0_net.parameters(), lr=3e-4)\n",
    "        self.q1_optimizer = optim.Adam(self.q1_net.parameters(), lr=3e-4)\n",
    "\n",
    "    def build_net(self, input_size, hidden_sizes, output_size=1,\n",
    "            output_activator=None):\n",
    "        layers = []\n",
    "        for input_size, output_size in zip(\n",
    "                [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):\n",
    "            layers.append(nn.Linear(input_size, output_size))\n",
    "            layers.append(nn.ReLU())\n",
    "        layers = layers[:-1]\n",
    "        if output_activator:\n",
    "            layers.append(output_activator)\n",
    "        net = nn.Sequential(*layers)\n",
    "        return net\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        state_tensor = torch.as_tensor(observation, dtype=torch.float).unsqueeze(0)\n",
    "        prob_tensor = self.actor_net(state_tensor)\n",
    "        action_tensor = distributions.Categorical(prob_tensor).sample()\n",
    "        action = action_tensor.numpy()[0]\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                state, _, _, action, next_state, reward, done, _ = \\\n",
    "                        self.trajectory[-8:]\n",
    "                self.replayer.store(state, action, reward, next_state, done)\n",
    "            if self.replayer.count >= 500:\n",
    "                self.learn()\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def update_net(self, target_net, evaluate_net, learning_rate=0.0025):\n",
    "        for target_param, evaluate_param in zip(\n",
    "                target_net.parameters(), evaluate_net.parameters()):\n",
    "            target_param.data.copy_(learning_rate * evaluate_param.data\n",
    "                    + (1 - learning_rate) * target_param.data)\n",
    "\n",
    "    def learn(self):\n",
    "        states, actions, rewards, next_states, dones = self.replayer.sample(128)\n",
    "        state_tensor = torch.as_tensor(states, dtype=torch.float)\n",
    "        action_tensor = torch.as_tensor(actions, dtype=torch.long)\n",
    "        reward_tensor = torch.as_tensor(rewards, dtype=torch.float)\n",
    "        next_state_tensor = torch.as_tensor(next_states, dtype=torch.float)\n",
    "        done_tensor = torch.as_tensor(dones, dtype=torch.float)\n",
    "\n",
    "        # train alpha\n",
    "        prob_tensor = self.actor_net(state_tensor)\n",
    "        ln_prob_tensor = torch.log(prob_tensor.clamp(1e-6, 1))\n",
    "        neg_entropy_tensor = (prob_tensor * ln_prob_tensor).sum()\n",
    "        # OR neg_entropy_tensor = torch.xlogy(prob_tensor, prob_tensor).sum()\n",
    "        grad_tensor = neg_entropy_tensor + self.target_entropy\n",
    "        alpha_loss_tensor = -self.ln_alpha_tensor * grad_tensor.detach()\n",
    "        self.alpha_optimizer.zero_grad()\n",
    "        alpha_loss_tensor.backward()\n",
    "        self.alpha_optimizer.step()\n",
    "\n",
    "        # train Q critic\n",
    "        next_v_tensor = self.v_target_net(next_state_tensor)\n",
    "        q_target_tensor = reward_tensor.unsqueeze(1) + self.gamma * \\\n",
    "                (1. - done_tensor.unsqueeze(1)) * next_v_tensor\n",
    "\n",
    "        all_q0_pred_tensor = self.q0_net(state_tensor)\n",
    "        q0_pred_tensor = torch.gather(all_q0_pred_tensor, 1, action_tensor.unsqueeze(1))\n",
    "        q0_loss_tensor = self.q0_loss(q0_pred_tensor, q_target_tensor.detach())\n",
    "        self.q0_optimizer.zero_grad()\n",
    "        q0_loss_tensor.backward()\n",
    "        self.q0_optimizer.step()\n",
    "\n",
    "        all_q1_pred_tensor = self.q1_net(state_tensor)\n",
    "        q1_pred_tensor = torch.gather(all_q1_pred_tensor, 1, action_tensor.unsqueeze(1))\n",
    "        q1_loss_tensor = self.q1_loss(q1_pred_tensor, q_target_tensor.detach())\n",
    "        self.q1_optimizer.zero_grad()\n",
    "        q1_loss_tensor.backward()\n",
    "        self.q1_optimizer.step()\n",
    "\n",
    "        # train V critic\n",
    "        q0_tensor = self.q0_net(state_tensor)\n",
    "        q1_tensor = self.q1_net(state_tensor)\n",
    "        q01_tensor = torch.min(q0_tensor, q1_tensor)\n",
    "        prob_tensor = self.actor_net(state_tensor)\n",
    "        ln_prob_tensor = torch.log(prob_tensor.clamp(1e-6, 1.))\n",
    "        alpha = self.ln_alpha_tensor.exp().detach().item()\n",
    "        entropic_q01_tensor = prob_tensor * (q01_tensor - alpha * ln_prob_tensor)\n",
    "        # OR entropic_q01_tensor = prob_tensor * (q01_tensor -\n",
    "        #         alpha * torch.xlogy(prob_tensor, prob_tensor)\n",
    "        v_target_tensor = torch.sum(entropic_q01_tensor, dim=-1, keepdim=True)\n",
    "        v_pred_tensor = self.v_evaluate_net(state_tensor)\n",
    "        v_loss_tensor = self.v_loss(v_pred_tensor, v_target_tensor.detach())\n",
    "        self.v_optimizer.zero_grad()\n",
    "        v_loss_tensor.backward()\n",
    "        self.v_optimizer.step()\n",
    "\n",
    "        self.update_net(self.v_target_net, self.v_evaluate_net)\n",
    "\n",
    "        # train actor\n",
    "        prob_q_tensor = prob_tensor * (alpha * ln_prob_tensor - q0_tensor)\n",
    "        actor_loss_tensor = prob_q_tensor.sum(axis=-1).mean()\n",
    "        self.actor_optimizer.zero_grad()\n",
    "        actor_loss_tensor.backward()\n",
    "        self.actor_optimizer.step()\n",
    "\n",
    "\n",
    "agent = SACAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "08:49:53 [INFO] ==== train ====\n",
      "08:49:53 [DEBUG] train episode 0: reward = -119.74, steps = 100\n",
      "08:49:53 [DEBUG] train episode 1: reward = -53.41, steps = 63\n",
      "08:49:53 [DEBUG] train episode 2: reward = -136.45, steps = 87\n",
      "08:49:53 [DEBUG] train episode 3: reward = -326.87, steps = 70\n",
      "08:49:54 [DEBUG] train episode 4: reward = -117.95, steps = 113\n",
      "08:49:55 [DEBUG] train episode 5: reward = -336.63, steps = 96\n",
      "08:49:58 [DEBUG] train episode 6: reward = -181.90, steps = 99\n",
      "08:50:02 [DEBUG] train episode 7: reward = -145.90, steps = 117\n",
      "08:50:05 [DEBUG] train episode 8: reward = -310.74, steps = 100\n",
      "08:50:07 [DEBUG] train episode 9: reward = -144.71, steps = 66\n",
      "08:50:11 [DEBUG] train episode 10: reward = -2.77, steps = 150\n",
      "08:50:14 [DEBUG] train episode 11: reward = -162.35, steps = 91\n",
      "08:50:16 [DEBUG] train episode 12: reward = -315.83, steps = 72\n",
      "08:50:18 [DEBUG] train episode 13: reward = -447.24, steps = 69\n",
      "08:50:21 [DEBUG] train episode 14: reward = -351.69, steps = 106\n",
      "08:50:23 [DEBUG] train episode 15: reward = -59.90, steps = 78\n",
      "08:50:26 [DEBUG] train episode 16: reward = -429.55, steps = 89\n",
      "08:50:30 [DEBUG] train episode 17: reward = -103.32, steps = 135\n",
      "08:50:33 [DEBUG] train episode 18: reward = -181.24, steps = 84\n",
      "08:50:36 [DEBUG] train episode 19: reward = -155.00, steps = 76\n",
      "08:50:40 [DEBUG] train episode 20: reward = -376.96, steps = 149\n",
      "08:50:43 [DEBUG] train episode 21: reward = -251.47, steps = 102\n",
      "08:50:46 [DEBUG] train episode 22: reward = -122.12, steps = 75\n",
      "08:50:49 [DEBUG] train episode 23: reward = -192.81, steps = 113\n",
      "08:50:54 [DEBUG] train episode 24: reward = -298.56, steps = 144\n",
      "08:50:58 [DEBUG] train episode 25: reward = -326.07, steps = 124\n",
      "08:51:01 [DEBUG] train episode 26: reward = -322.35, steps = 88\n",
      "08:51:05 [DEBUG] train episode 27: reward = -275.14, steps = 117\n",
      "08:51:07 [DEBUG] train episode 28: reward = -60.64, steps = 88\n",
      "08:51:11 [DEBUG] train episode 29: reward = -227.72, steps = 100\n",
      "08:51:13 [DEBUG] train episode 30: reward = -67.02, steps = 92\n",
      "08:51:16 [DEBUG] train episode 31: reward = -254.04, steps = 83\n",
      "08:51:19 [DEBUG] train episode 32: reward = -349.88, steps = 107\n",
      "08:51:22 [DEBUG] train episode 33: reward = -37.67, steps = 81\n",
      "08:51:24 [DEBUG] train episode 34: reward = -77.48, steps = 72\n",
      "08:51:27 [DEBUG] train episode 35: reward = -55.36, steps = 99\n",
      "08:51:30 [DEBUG] train episode 36: reward = -90.90, steps = 78\n",
      "08:51:33 [DEBUG] train episode 37: reward = -284.01, steps = 103\n",
      "08:51:35 [DEBUG] train episode 38: reward = -188.93, steps = 62\n",
      "08:51:38 [DEBUG] train episode 39: reward = -155.66, steps = 90\n",
      "08:51:40 [DEBUG] train episode 40: reward = -153.28, steps = 81\n",
      "08:51:42 [DEBUG] train episode 41: reward = -100.22, steps = 58\n",
      "08:51:45 [DEBUG] train episode 42: reward = -77.63, steps = 94\n",
      "08:51:48 [DEBUG] train episode 43: reward = -320.48, steps = 81\n",
      "08:51:49 [DEBUG] train episode 44: reward = -107.22, steps = 54\n",
      "08:51:52 [DEBUG] train episode 45: reward = -111.05, steps = 74\n",
      "08:51:55 [DEBUG] train episode 46: reward = -266.53, steps = 118\n",
      "08:51:59 [DEBUG] train episode 47: reward = -99.81, steps = 97\n",
      "08:52:02 [DEBUG] train episode 48: reward = -280.52, steps = 114\n",
      "08:52:04 [DEBUG] train episode 49: reward = -89.81, steps = 69\n",
      "08:52:07 [DEBUG] train episode 50: reward = -229.07, steps = 99\n",
      "08:52:10 [DEBUG] train episode 51: reward = -6.66, steps = 79\n",
      "08:52:13 [DEBUG] train episode 52: reward = -117.21, steps = 88\n",
      "08:52:15 [DEBUG] train episode 53: reward = -106.00, steps = 75\n",
      "08:52:17 [DEBUG] train episode 54: reward = -175.23, steps = 76\n",
      "08:52:20 [DEBUG] train episode 55: reward = -159.31, steps = 90\n",
      "08:52:23 [DEBUG] train episode 56: reward = -37.40, steps = 95\n",
      "08:52:26 [DEBUG] train episode 57: reward = -146.62, steps = 94\n",
      "08:52:29 [DEBUG] train episode 58: reward = -273.68, steps = 87\n",
      "08:52:32 [DEBUG] train episode 59: reward = -58.22, steps = 92\n",
      "08:52:36 [DEBUG] train episode 60: reward = -182.84, steps = 124\n",
      "08:52:39 [DEBUG] train episode 61: reward = -129.32, steps = 102\n",
      "08:52:43 [DEBUG] train episode 62: reward = -53.85, steps = 119\n",
      "08:52:46 [DEBUG] train episode 63: reward = -220.74, steps = 90\n",
      "08:52:49 [DEBUG] train episode 64: reward = -165.93, steps = 110\n",
      "08:52:52 [DEBUG] train episode 65: reward = -175.62, steps = 75\n",
      "08:52:55 [DEBUG] train episode 66: reward = -36.51, steps = 90\n",
      "08:52:59 [DEBUG] train episode 67: reward = -210.49, steps = 103\n",
      "08:53:04 [DEBUG] train episode 68: reward = -73.05, steps = 154\n",
      "08:53:07 [DEBUG] train episode 69: reward = -151.31, steps = 102\n",
      "08:53:10 [DEBUG] train episode 70: reward = -82.39, steps = 87\n",
      "08:53:15 [DEBUG] train episode 71: reward = -381.80, steps = 174\n",
      "08:53:19 [DEBUG] train episode 72: reward = -319.34, steps = 114\n",
      "08:53:23 [DEBUG] train episode 73: reward = -19.32, steps = 125\n",
      "08:53:27 [DEBUG] train episode 74: reward = -195.74, steps = 125\n",
      "08:53:31 [DEBUG] train episode 75: reward = 23.41, steps = 128\n",
      "08:53:33 [DEBUG] train episode 76: reward = -120.23, steps = 83\n",
      "08:53:36 [DEBUG] train episode 77: reward = -18.84, steps = 89\n",
      "08:53:40 [DEBUG] train episode 78: reward = -101.72, steps = 111\n",
      "08:53:43 [DEBUG] train episode 79: reward = -92.64, steps = 102\n",
      "08:53:46 [DEBUG] train episode 80: reward = -140.65, steps = 97\n",
      "08:53:53 [DEBUG] train episode 81: reward = -54.77, steps = 226\n",
      "08:53:57 [DEBUG] train episode 82: reward = -236.75, steps = 123\n",
      "08:54:01 [DEBUG] train episode 83: reward = -30.38, steps = 112\n",
      "08:54:03 [DEBUG] train episode 84: reward = -88.70, steps = 76\n",
      "08:54:08 [DEBUG] train episode 85: reward = -207.05, steps = 158\n",
      "08:54:12 [DEBUG] train episode 86: reward = -96.84, steps = 125\n",
      "08:54:16 [DEBUG] train episode 87: reward = -219.92, steps = 118\n",
      "08:54:20 [DEBUG] train episode 88: reward = -79.93, steps = 116\n",
      "08:54:27 [DEBUG] train episode 89: reward = -80.47, steps = 232\n",
      "08:54:32 [DEBUG] train episode 90: reward = -179.80, steps = 138\n",
      "08:54:36 [DEBUG] train episode 91: reward = -238.42, steps = 119\n",
      "08:54:41 [DEBUG] train episode 92: reward = -198.85, steps = 164\n",
      "08:54:48 [DEBUG] train episode 93: reward = -221.22, steps = 236\n",
      "08:54:57 [DEBUG] train episode 94: reward = -74.74, steps = 274\n",
      "08:55:01 [DEBUG] train episode 95: reward = -257.51, steps = 119\n",
      "08:55:08 [DEBUG] train episode 96: reward = -177.59, steps = 236\n",
      "08:55:14 [DEBUG] train episode 97: reward = -412.11, steps = 181\n",
      "08:55:17 [DEBUG] train episode 98: reward = -207.53, steps = 118\n",
      "08:55:28 [DEBUG] train episode 99: reward = -240.19, steps = 312\n",
      "08:55:34 [DEBUG] train episode 100: reward = 9.24, steps = 182\n",
      "08:55:58 [DEBUG] train episode 101: reward = -382.12, steps = 765\n",
      "08:56:02 [DEBUG] train episode 102: reward = -108.51, steps = 107\n",
      "08:56:08 [DEBUG] train episode 103: reward = -328.58, steps = 207\n",
      "08:56:14 [DEBUG] train episode 104: reward = -258.20, steps = 180\n",
      "08:56:18 [DEBUG] train episode 105: reward = -314.33, steps = 114\n",
      "08:56:24 [DEBUG] train episode 106: reward = -322.31, steps = 190\n",
      "08:56:37 [DEBUG] train episode 107: reward = -206.53, steps = 395\n",
      "08:56:45 [DEBUG] train episode 108: reward = -98.26, steps = 256\n",
      "08:57:13 [DEBUG] train episode 109: reward = -174.85, steps = 877\n",
      "08:57:22 [DEBUG] train episode 110: reward = -26.02, steps = 255\n",
      "08:57:27 [DEBUG] train episode 111: reward = -17.03, steps = 155\n",
      "08:57:32 [DEBUG] train episode 112: reward = -75.25, steps = 169\n",
      "08:57:40 [DEBUG] train episode 113: reward = 56.28, steps = 252\n",
      "08:57:46 [DEBUG] train episode 114: reward = -56.24, steps = 187\n",
      "08:57:55 [DEBUG] train episode 115: reward = 41.91, steps = 293\n",
      "08:58:02 [DEBUG] train episode 116: reward = -94.69, steps = 211\n",
      "08:58:09 [DEBUG] train episode 117: reward = -127.31, steps = 190\n",
      "08:58:16 [DEBUG] train episode 118: reward = -142.80, steps = 240\n",
      "08:58:23 [DEBUG] train episode 119: reward = -124.74, steps = 211\n",
      "08:58:33 [DEBUG] train episode 120: reward = -60.55, steps = 320\n",
      "08:58:41 [DEBUG] train episode 121: reward = -125.80, steps = 249\n",
      "08:58:52 [DEBUG] train episode 122: reward = -86.13, steps = 355\n",
      "08:58:59 [DEBUG] train episode 123: reward = -137.90, steps = 198\n",
      "08:59:12 [DEBUG] train episode 124: reward = -46.64, steps = 428\n",
      "08:59:22 [DEBUG] train episode 125: reward = -105.46, steps = 296\n",
      "08:59:28 [DEBUG] train episode 126: reward = -137.83, steps = 184\n",
      "08:59:37 [DEBUG] train episode 127: reward = -120.26, steps = 277\n",
      "08:59:48 [DEBUG] train episode 128: reward = -44.16, steps = 365\n",
      "09:00:04 [DEBUG] train episode 129: reward = -170.22, steps = 478\n",
      "09:00:13 [DEBUG] train episode 130: reward = -112.95, steps = 290\n",
      "09:00:17 [DEBUG] train episode 131: reward = -134.92, steps = 115\n",
      "09:00:26 [DEBUG] train episode 132: reward = -148.71, steps = 281\n",
      "09:00:32 [DEBUG] train episode 133: reward = -87.77, steps = 183\n",
      "09:00:50 [DEBUG] train episode 134: reward = -179.98, steps = 548\n",
      "09:00:54 [DEBUG] train episode 135: reward = -86.95, steps = 150\n",
      "09:01:08 [DEBUG] train episode 136: reward = -185.02, steps = 431\n",
      "09:01:21 [DEBUG] train episode 137: reward = -107.17, steps = 407\n",
      "09:01:35 [DEBUG] train episode 138: reward = -217.35, steps = 427\n",
      "09:01:40 [DEBUG] train episode 139: reward = -115.51, steps = 157\n",
      "09:01:59 [DEBUG] train episode 140: reward = -239.48, steps = 602\n",
      "09:02:09 [DEBUG] train episode 141: reward = -118.03, steps = 307\n",
      "09:02:14 [DEBUG] train episode 142: reward = -94.33, steps = 162\n",
      "09:02:27 [DEBUG] train episode 143: reward = -151.36, steps = 408\n",
      "09:02:33 [DEBUG] train episode 144: reward = -113.35, steps = 194\n",
      "09:03:06 [DEBUG] train episode 145: reward = -80.29, steps = 1000\n",
      "09:03:15 [DEBUG] train episode 146: reward = -114.29, steps = 307\n",
      "09:03:30 [DEBUG] train episode 147: reward = -123.43, steps = 447\n",
      "09:03:37 [DEBUG] train episode 148: reward = -72.17, steps = 230\n",
      "09:03:56 [DEBUG] train episode 149: reward = -84.20, steps = 594\n",
      "09:04:17 [DEBUG] train episode 150: reward = -188.51, steps = 649\n",
      "09:04:24 [DEBUG] train episode 151: reward = -90.44, steps = 227\n",
      "09:04:58 [DEBUG] train episode 152: reward = 49.20, steps = 1000\n",
      "09:05:24 [DEBUG] train episode 153: reward = -176.41, steps = 791\n",
      "09:05:52 [DEBUG] train episode 154: reward = -214.44, steps = 873\n",
      "09:06:25 [DEBUG] train episode 155: reward = -117.69, steps = 1000\n",
      "09:06:59 [DEBUG] train episode 156: reward = -79.93, steps = 1000\n",
      "09:07:34 [DEBUG] train episode 157: reward = -125.30, steps = 1000\n",
      "09:08:08 [DEBUG] train episode 158: reward = -77.60, steps = 1000\n",
      "09:08:41 [DEBUG] train episode 159: reward = -66.79, steps = 1000\n",
      "09:09:15 [DEBUG] train episode 160: reward = -54.37, steps = 1000\n",
      "09:09:31 [DEBUG] train episode 161: reward = -91.15, steps = 488\n",
      "09:10:05 [DEBUG] train episode 162: reward = -70.56, steps = 1000\n",
      "09:10:39 [DEBUG] train episode 163: reward = -18.33, steps = 1000\n",
      "09:11:12 [DEBUG] train episode 164: reward = 1.63, steps = 1000\n",
      "09:11:44 [DEBUG] train episode 165: reward = -39.78, steps = 1000\n",
      "09:12:18 [DEBUG] train episode 166: reward = 2.87, steps = 1000\n",
      "09:12:50 [DEBUG] train episode 167: reward = -26.18, steps = 1000\n",
      "09:13:23 [DEBUG] train episode 168: reward = -71.67, steps = 1000\n",
      "09:13:57 [DEBUG] train episode 169: reward = -36.45, steps = 1000\n",
      "09:14:30 [DEBUG] train episode 170: reward = -41.07, steps = 1000\n",
      "09:15:04 [DEBUG] train episode 171: reward = -31.90, steps = 1000\n",
      "09:15:37 [DEBUG] train episode 172: reward = -73.11, steps = 1000\n",
      "09:16:12 [DEBUG] train episode 173: reward = -44.80, steps = 1000\n",
      "09:16:47 [DEBUG] train episode 174: reward = -12.23, steps = 1000\n",
      "09:17:21 [DEBUG] train episode 175: reward = -6.38, steps = 1000\n",
      "09:17:56 [DEBUG] train episode 176: reward = -4.75, steps = 1000\n",
      "09:18:31 [DEBUG] train episode 177: reward = -17.49, steps = 1000\n",
      "09:19:05 [DEBUG] train episode 178: reward = -31.29, steps = 1000\n",
      "09:19:40 [DEBUG] train episode 179: reward = -2.97, steps = 1000\n",
      "09:20:15 [DEBUG] train episode 180: reward = 12.63, steps = 1000\n",
      "09:20:51 [DEBUG] train episode 181: reward = 4.95, steps = 1000\n",
      "09:21:27 [DEBUG] train episode 182: reward = 23.84, steps = 1000\n",
      "09:22:02 [DEBUG] train episode 183: reward = 5.81, steps = 1000\n",
      "09:22:40 [DEBUG] train episode 184: reward = -25.18, steps = 1000\n",
      "09:23:15 [DEBUG] train episode 185: reward = -12.47, steps = 1000\n",
      "09:23:52 [DEBUG] train episode 186: reward = -75.48, steps = 1000\n",
      "09:24:28 [DEBUG] train episode 187: reward = -11.71, steps = 1000\n",
      "09:25:04 [DEBUG] train episode 188: reward = -32.70, steps = 1000\n",
      "09:25:41 [DEBUG] train episode 189: reward = 14.71, steps = 1000\n",
      "09:26:18 [DEBUG] train episode 190: reward = 4.37, steps = 1000\n",
      "09:26:55 [DEBUG] train episode 191: reward = -25.78, steps = 1000\n",
      "09:27:33 [DEBUG] train episode 192: reward = -21.27, steps = 1000\n",
      "09:28:11 [DEBUG] train episode 193: reward = -49.02, steps = 1000\n",
      "09:28:48 [DEBUG] train episode 194: reward = -14.77, steps = 1000\n",
      "09:29:27 [DEBUG] train episode 195: reward = -36.21, steps = 1000\n",
      "09:30:05 [DEBUG] train episode 196: reward = 0.20, steps = 1000\n",
      "09:30:44 [DEBUG] train episode 197: reward = 19.84, steps = 1000\n",
      "09:31:22 [DEBUG] train episode 198: reward = -30.18, steps = 1000\n",
      "09:32:02 [DEBUG] train episode 199: reward = 6.53, steps = 1000\n",
      "09:32:41 [DEBUG] train episode 200: reward = 7.37, steps = 1000\n",
      "09:33:20 [DEBUG] train episode 201: reward = 13.28, steps = 1000\n",
      "09:34:01 [DEBUG] train episode 202: reward = 7.84, steps = 1000\n",
      "09:34:40 [DEBUG] train episode 203: reward = 11.01, steps = 1000\n",
      "09:35:20 [DEBUG] train episode 204: reward = -1.73, steps = 1000\n",
      "09:36:01 [DEBUG] train episode 205: reward = -55.33, steps = 1000\n",
      "09:36:41 [DEBUG] train episode 206: reward = -37.24, steps = 1000\n",
      "09:37:21 [DEBUG] train episode 207: reward = -67.50, steps = 1000\n",
      "09:38:01 [DEBUG] train episode 208: reward = 3.71, steps = 1000\n",
      "09:38:42 [DEBUG] train episode 209: reward = -39.68, steps = 1000\n",
      "09:39:13 [DEBUG] train episode 210: reward = 168.03, steps = 794\n",
      "09:39:52 [DEBUG] train episode 211: reward = -36.16, steps = 1000\n",
      "09:40:33 [DEBUG] train episode 212: reward = -54.07, steps = 1000\n",
      "09:41:11 [DEBUG] train episode 213: reward = 28.84, steps = 1000\n",
      "09:41:50 [DEBUG] train episode 214: reward = 8.41, steps = 1000\n",
      "09:41:53 [DEBUG] train episode 215: reward = -190.62, steps = 84\n",
      "09:42:33 [DEBUG] train episode 216: reward = -45.12, steps = 1000\n",
      "09:43:12 [DEBUG] train episode 217: reward = -6.79, steps = 1000\n",
      "09:43:15 [DEBUG] train episode 218: reward = -52.57, steps = 72\n",
      "09:43:54 [DEBUG] train episode 219: reward = 7.98, steps = 1000\n",
      "09:44:33 [DEBUG] train episode 220: reward = -60.27, steps = 1000\n",
      "09:45:14 [DEBUG] train episode 221: reward = -34.45, steps = 1000\n",
      "09:45:23 [DEBUG] train episode 222: reward = -15.16, steps = 256\n",
      "09:45:31 [DEBUG] train episode 223: reward = -40.23, steps = 209\n",
      "09:46:10 [DEBUG] train episode 224: reward = -40.07, steps = 1000\n",
      "09:46:49 [DEBUG] train episode 225: reward = -30.90, steps = 1000\n",
      "09:47:28 [DEBUG] train episode 226: reward = -65.34, steps = 1000\n",
      "09:48:08 [DEBUG] train episode 227: reward = 0.33, steps = 1000\n",
      "09:48:47 [DEBUG] train episode 228: reward = -0.08, steps = 1000\n",
      "09:49:25 [DEBUG] train episode 229: reward = -23.53, steps = 1000\n",
      "09:50:04 [DEBUG] train episode 230: reward = -32.84, steps = 1000\n",
      "09:50:42 [DEBUG] train episode 231: reward = 8.43, steps = 1000\n",
      "09:51:21 [DEBUG] train episode 232: reward = 22.71, steps = 1000\n",
      "09:52:00 [DEBUG] train episode 233: reward = 5.20, steps = 1000\n",
      "09:52:41 [DEBUG] train episode 234: reward = -53.36, steps = 1000\n",
      "09:53:22 [DEBUG] train episode 235: reward = 1.53, steps = 1000\n",
      "09:54:01 [DEBUG] train episode 236: reward = 44.41, steps = 1000\n",
      "09:54:41 [DEBUG] train episode 237: reward = -44.29, steps = 1000\n",
      "09:55:17 [DEBUG] train episode 238: reward = -172.51, steps = 905\n",
      "09:55:57 [DEBUG] train episode 239: reward = -29.64, steps = 1000\n",
      "09:56:36 [DEBUG] train episode 240: reward = -64.17, steps = 1000\n",
      "09:57:16 [DEBUG] train episode 241: reward = -24.15, steps = 1000\n",
      "09:57:43 [DEBUG] train episode 242: reward = -125.58, steps = 694\n",
      "09:58:22 [DEBUG] train episode 243: reward = -27.99, steps = 1000\n",
      "09:59:01 [DEBUG] train episode 244: reward = -6.29, steps = 1000\n",
      "09:59:40 [DEBUG] train episode 245: reward = -24.44, steps = 1000\n",
      "10:00:18 [DEBUG] train episode 246: reward = -1.26, steps = 1000\n",
      "10:00:57 [DEBUG] train episode 247: reward = 11.06, steps = 1000\n",
      "10:01:36 [DEBUG] train episode 248: reward = -63.15, steps = 1000\n",
      "10:02:15 [DEBUG] train episode 249: reward = -44.60, steps = 1000\n",
      "10:02:54 [DEBUG] train episode 250: reward = 18.97, steps = 1000\n",
      "10:03:33 [DEBUG] train episode 251: reward = -31.76, steps = 1000\n",
      "10:04:13 [DEBUG] train episode 252: reward = -42.31, steps = 1000\n",
      "10:04:16 [DEBUG] train episode 253: reward = -149.63, steps = 92\n",
      "10:04:56 [DEBUG] train episode 254: reward = -48.89, steps = 1000\n",
      "10:05:35 [DEBUG] train episode 255: reward = -5.55, steps = 1000\n",
      "10:06:14 [DEBUG] train episode 256: reward = -30.94, steps = 1000\n",
      "10:06:53 [DEBUG] train episode 257: reward = -9.88, steps = 1000\n",
      "10:07:32 [DEBUG] train episode 258: reward = -41.72, steps = 1000\n",
      "10:08:12 [DEBUG] train episode 259: reward = 86.70, steps = 987\n",
      "10:08:52 [DEBUG] train episode 260: reward = -39.33, steps = 1000\n",
      "10:09:32 [DEBUG] train episode 261: reward = 7.94, steps = 1000\n",
      "10:10:12 [DEBUG] train episode 262: reward = -6.40, steps = 1000\n",
      "10:10:51 [DEBUG] train episode 263: reward = -27.86, steps = 1000\n",
      "10:11:30 [DEBUG] train episode 264: reward = -71.48, steps = 1000\n",
      "10:12:10 [DEBUG] train episode 265: reward = -21.86, steps = 1000\n",
      "10:12:49 [DEBUG] train episode 266: reward = -45.56, steps = 1000\n",
      "10:13:29 [DEBUG] train episode 267: reward = -64.30, steps = 1000\n",
      "10:14:09 [DEBUG] train episode 268: reward = 0.52, steps = 1000\n",
      "10:14:49 [DEBUG] train episode 269: reward = -20.07, steps = 1000\n",
      "10:15:30 [DEBUG] train episode 270: reward = -28.48, steps = 1000\n",
      "10:16:10 [DEBUG] train episode 271: reward = -23.65, steps = 1000\n",
      "10:16:50 [DEBUG] train episode 272: reward = -1.27, steps = 1000\n",
      "10:17:31 [DEBUG] train episode 273: reward = 3.31, steps = 1000\n",
      "10:18:12 [DEBUG] train episode 274: reward = -1.39, steps = 1000\n",
      "10:18:52 [DEBUG] train episode 275: reward = -32.75, steps = 1000\n",
      "10:19:34 [DEBUG] train episode 276: reward = -39.80, steps = 1000\n",
      "10:20:14 [DEBUG] train episode 277: reward = -24.09, steps = 1000\n",
      "10:20:55 [DEBUG] train episode 278: reward = -39.08, steps = 1000\n",
      "10:21:36 [DEBUG] train episode 279: reward = -47.51, steps = 1000\n",
      "10:22:16 [DEBUG] train episode 280: reward = -23.01, steps = 1000\n",
      "10:23:00 [DEBUG] train episode 281: reward = -27.71, steps = 1000\n",
      "10:23:40 [DEBUG] train episode 282: reward = 3.29, steps = 1000\n",
      "10:24:21 [DEBUG] train episode 283: reward = -18.14, steps = 1000\n",
      "10:25:02 [DEBUG] train episode 284: reward = -47.98, steps = 1000\n",
      "10:25:43 [DEBUG] train episode 285: reward = -20.43, steps = 1000\n",
      "10:26:24 [DEBUG] train episode 286: reward = -8.02, steps = 1000\n",
      "10:27:05 [DEBUG] train episode 287: reward = -27.27, steps = 1000\n",
      "10:27:47 [DEBUG] train episode 288: reward = -56.75, steps = 1000\n",
      "10:28:27 [DEBUG] train episode 289: reward = 9.80, steps = 1000\n",
      "10:29:08 [DEBUG] train episode 290: reward = -55.58, steps = 1000\n",
      "10:29:50 [DEBUG] train episode 291: reward = -43.32, steps = 1000\n",
      "10:30:32 [DEBUG] train episode 292: reward = -27.83, steps = 1000\n",
      "10:30:38 [DEBUG] train episode 293: reward = -77.91, steps = 145\n",
      "10:31:18 [DEBUG] train episode 294: reward = -59.44, steps = 1000\n",
      "10:31:59 [DEBUG] train episode 295: reward = -4.06, steps = 1000\n",
      "10:32:40 [DEBUG] train episode 296: reward = 11.90, steps = 1000\n",
      "10:33:20 [DEBUG] train episode 297: reward = -49.49, steps = 1000\n",
      "10:34:01 [DEBUG] train episode 298: reward = 7.90, steps = 1000\n",
      "10:34:42 [DEBUG] train episode 299: reward = -63.10, steps = 1000\n",
      "10:35:22 [DEBUG] train episode 300: reward = -16.20, steps = 1000\n",
      "10:36:03 [DEBUG] train episode 301: reward = -1.74, steps = 1000\n",
      "10:36:44 [DEBUG] train episode 302: reward = -35.07, steps = 1000\n",
      "10:36:49 [DEBUG] train episode 303: reward = -6.43, steps = 144\n",
      "10:37:30 [DEBUG] train episode 304: reward = 3.82, steps = 1000\n",
      "10:38:10 [DEBUG] train episode 305: reward = -13.07, steps = 1000\n",
      "10:38:51 [DEBUG] train episode 306: reward = -21.62, steps = 1000\n",
      "10:39:32 [DEBUG] train episode 307: reward = -37.21, steps = 1000\n",
      "10:40:14 [DEBUG] train episode 308: reward = -5.40, steps = 1000\n",
      "10:40:19 [DEBUG] train episode 309: reward = -47.19, steps = 136\n",
      "10:41:01 [DEBUG] train episode 310: reward = -34.44, steps = 1000\n",
      "10:41:06 [DEBUG] train episode 311: reward = -53.53, steps = 126\n",
      "10:41:47 [DEBUG] train episode 312: reward = 8.08, steps = 1000\n",
      "10:42:28 [DEBUG] train episode 313: reward = -22.24, steps = 1000\n",
      "10:43:09 [DEBUG] train episode 314: reward = -18.65, steps = 1000\n",
      "10:43:50 [DEBUG] train episode 315: reward = -9.58, steps = 1000\n",
      "10:44:21 [DEBUG] train episode 316: reward = 200.26, steps = 753\n",
      "10:45:02 [DEBUG] train episode 317: reward = -30.10, steps = 1000\n",
      "10:45:42 [DEBUG] train episode 318: reward = -60.28, steps = 1000\n",
      "10:46:22 [DEBUG] train episode 319: reward = 17.45, steps = 1000\n",
      "10:47:03 [DEBUG] train episode 320: reward = -33.42, steps = 1000\n",
      "10:47:44 [DEBUG] train episode 321: reward = -19.99, steps = 1000\n",
      "10:48:25 [DEBUG] train episode 322: reward = 13.88, steps = 1000\n",
      "10:49:05 [DEBUG] train episode 323: reward = -35.00, steps = 1000\n",
      "10:49:47 [DEBUG] train episode 324: reward = -0.80, steps = 1000\n",
      "10:49:53 [DEBUG] train episode 325: reward = -31.59, steps = 174\n",
      "10:49:58 [DEBUG] train episode 326: reward = -93.30, steps = 124\n",
      "10:50:38 [DEBUG] train episode 327: reward = -4.55, steps = 1000\n",
      "10:51:18 [DEBUG] train episode 328: reward = -22.92, steps = 1000\n",
      "10:51:59 [DEBUG] train episode 329: reward = -66.67, steps = 1000\n",
      "10:52:06 [DEBUG] train episode 330: reward = -91.46, steps = 199\n",
      "10:52:48 [DEBUG] train episode 331: reward = 13.11, steps = 1000\n",
      "10:53:28 [DEBUG] train episode 332: reward = 19.52, steps = 1000\n",
      "10:54:09 [DEBUG] train episode 333: reward = -23.95, steps = 1000\n",
      "10:54:50 [DEBUG] train episode 334: reward = -26.49, steps = 1000\n",
      "10:55:31 [DEBUG] train episode 335: reward = 15.47, steps = 1000\n",
      "10:56:13 [DEBUG] train episode 336: reward = -17.99, steps = 1000\n",
      "10:56:53 [DEBUG] train episode 337: reward = -14.47, steps = 1000\n",
      "10:57:34 [DEBUG] train episode 338: reward = -58.33, steps = 1000\n",
      "10:58:14 [DEBUG] train episode 339: reward = -11.58, steps = 1000\n",
      "10:58:45 [DEBUG] train episode 340: reward = -90.19, steps = 769\n",
      "10:59:25 [DEBUG] train episode 341: reward = -11.28, steps = 1000\n",
      "11:00:06 [DEBUG] train episode 342: reward = 8.60, steps = 1000\n",
      "11:00:47 [DEBUG] train episode 343: reward = -26.04, steps = 1000\n",
      "11:01:27 [DEBUG] train episode 344: reward = 20.18, steps = 1000\n",
      "11:02:08 [DEBUG] train episode 345: reward = -22.66, steps = 1000\n",
      "11:02:49 [DEBUG] train episode 346: reward = -31.68, steps = 1000\n",
      "11:03:31 [DEBUG] train episode 347: reward = -26.41, steps = 1000\n",
      "11:04:11 [DEBUG] train episode 348: reward = -6.08, steps = 1000\n",
      "11:04:37 [DEBUG] train episode 349: reward = 146.24, steps = 651\n",
      "11:05:09 [DEBUG] train episode 350: reward = 155.83, steps = 780\n",
      "11:05:18 [DEBUG] train episode 351: reward = 7.82, steps = 245\n",
      "11:05:51 [DEBUG] train episode 352: reward = 80.29, steps = 809\n",
      "11:06:32 [DEBUG] train episode 353: reward = -36.24, steps = 1000\n",
      "11:07:04 [DEBUG] train episode 354: reward = -61.48, steps = 801\n",
      "11:07:46 [DEBUG] train episode 355: reward = -19.42, steps = 1000\n",
      "11:08:27 [DEBUG] train episode 356: reward = -27.06, steps = 1000\n",
      "11:09:07 [DEBUG] train episode 357: reward = -21.69, steps = 1000\n",
      "11:09:50 [DEBUG] train episode 358: reward = -25.17, steps = 1000\n",
      "11:10:29 [DEBUG] train episode 359: reward = 97.53, steps = 963\n",
      "11:11:10 [DEBUG] train episode 360: reward = -8.54, steps = 1000\n",
      "11:11:37 [DEBUG] train episode 361: reward = 185.19, steps = 682\n",
      "11:12:18 [DEBUG] train episode 362: reward = 23.42, steps = 1000\n",
      "11:12:59 [DEBUG] train episode 363: reward = 3.70, steps = 1000\n",
      "11:13:40 [DEBUG] train episode 364: reward = 40.67, steps = 1000\n",
      "11:14:20 [DEBUG] train episode 365: reward = -17.96, steps = 1000\n",
      "11:15:02 [DEBUG] train episode 366: reward = -22.07, steps = 1000\n",
      "11:15:26 [DEBUG] train episode 367: reward = 197.98, steps = 616\n",
      "11:16:07 [DEBUG] train episode 368: reward = -29.95, steps = 1000\n",
      "11:16:48 [DEBUG] train episode 369: reward = -14.75, steps = 1000\n",
      "11:17:29 [DEBUG] train episode 370: reward = -29.98, steps = 1000\n",
      "11:18:10 [DEBUG] train episode 371: reward = 17.82, steps = 1000\n",
      "11:18:52 [DEBUG] train episode 372: reward = -39.24, steps = 1000\n",
      "11:19:34 [DEBUG] train episode 373: reward = -39.12, steps = 1000\n",
      "11:20:15 [DEBUG] train episode 374: reward = -10.31, steps = 1000\n",
      "11:20:56 [DEBUG] train episode 375: reward = -15.00, steps = 1000\n",
      "11:21:21 [DEBUG] train episode 376: reward = 187.56, steps = 615\n",
      "11:21:33 [DEBUG] train episode 377: reward = 268.76, steps = 301\n",
      "11:22:14 [DEBUG] train episode 378: reward = -53.70, steps = 1000\n",
      "11:22:56 [DEBUG] train episode 379: reward = 39.52, steps = 1000\n",
      "11:23:31 [DEBUG] train episode 380: reward = 104.65, steps = 850\n",
      "11:24:12 [DEBUG] train episode 381: reward = -59.91, steps = 1000\n",
      "11:24:52 [DEBUG] train episode 382: reward = 21.33, steps = 1000\n",
      "11:25:33 [DEBUG] train episode 383: reward = -19.80, steps = 1000\n",
      "11:25:39 [DEBUG] train episode 384: reward = 28.69, steps = 151\n",
      "11:26:21 [DEBUG] train episode 385: reward = -57.24, steps = 1000\n",
      "11:26:30 [DEBUG] train episode 386: reward = 249.79, steps = 220\n",
      "11:26:36 [DEBUG] train episode 387: reward = 7.72, steps = 172\n",
      "11:27:18 [DEBUG] train episode 388: reward = 35.42, steps = 1000\n",
      "11:27:35 [DEBUG] train episode 389: reward = 239.86, steps = 453\n",
      "11:28:16 [DEBUG] train episode 390: reward = 70.65, steps = 1000\n",
      "11:28:35 [DEBUG] train episode 391: reward = 182.73, steps = 483\n",
      "11:29:16 [DEBUG] train episode 392: reward = 115.86, steps = 1000\n",
      "11:29:56 [DEBUG] train episode 393: reward = 106.90, steps = 1000\n",
      "11:30:01 [DEBUG] train episode 394: reward = 30.09, steps = 129\n",
      "11:30:08 [DEBUG] train episode 395: reward = 19.33, steps = 170\n",
      "11:30:18 [DEBUG] train episode 396: reward = 275.51, steps = 251\n",
      "11:30:24 [DEBUG] train episode 397: reward = 13.79, steps = 154\n",
      "11:30:30 [DEBUG] train episode 398: reward = 26.55, steps = 149\n",
      "11:30:33 [DEBUG] train episode 399: reward = -3.53, steps = 91\n",
      "11:30:35 [DEBUG] train episode 400: reward = -53.04, steps = 56\n",
      "11:30:41 [DEBUG] train episode 401: reward = -15.88, steps = 142\n",
      "11:30:46 [DEBUG] train episode 402: reward = 31.50, steps = 136\n",
      "11:31:26 [DEBUG] train episode 403: reward = 135.86, steps = 1000\n",
      "11:31:31 [DEBUG] train episode 404: reward = -1.47, steps = 108\n",
      "11:32:10 [DEBUG] train episode 405: reward = 85.22, steps = 1000\n",
      "11:32:16 [DEBUG] train episode 406: reward = 12.88, steps = 141\n",
      "11:32:22 [DEBUG] train episode 407: reward = 9.83, steps = 156\n",
      "11:32:31 [DEBUG] train episode 408: reward = 245.56, steps = 239\n",
      "11:32:38 [DEBUG] train episode 409: reward = 43.41, steps = 162\n",
      "11:33:18 [DEBUG] train episode 410: reward = 122.64, steps = 1000\n",
      "11:33:25 [DEBUG] train episode 411: reward = -3.57, steps = 182\n",
      "11:34:05 [DEBUG] train episode 412: reward = 143.42, steps = 1000\n",
      "11:34:09 [DEBUG] train episode 413: reward = -25.21, steps = 124\n",
      "11:34:15 [DEBUG] train episode 414: reward = -18.99, steps = 140\n",
      "11:34:54 [DEBUG] train episode 415: reward = 100.31, steps = 1000\n",
      "11:35:33 [DEBUG] train episode 416: reward = 114.46, steps = 1000\n",
      "11:35:41 [DEBUG] train episode 417: reward = 232.46, steps = 201\n",
      "11:35:55 [DEBUG] train episode 418: reward = 266.54, steps = 366\n",
      "11:36:34 [DEBUG] train episode 419: reward = 110.42, steps = 1000\n",
      "11:37:13 [DEBUG] train episode 420: reward = 121.11, steps = 1000\n",
      "11:37:19 [DEBUG] train episode 421: reward = -9.14, steps = 158\n",
      "11:37:27 [DEBUG] train episode 422: reward = 239.80, steps = 207\n",
      "11:37:45 [DEBUG] train episode 423: reward = 245.35, steps = 458\n",
      "11:37:53 [DEBUG] train episode 424: reward = 257.88, steps = 224\n",
      "11:38:32 [DEBUG] train episode 425: reward = 76.03, steps = 1000\n",
      "11:38:38 [DEBUG] train episode 426: reward = 241.15, steps = 163\n",
      "11:39:17 [DEBUG] train episode 427: reward = 121.12, steps = 1000\n",
      "11:39:22 [DEBUG] train episode 428: reward = 60.61, steps = 138\n",
      "11:39:31 [DEBUG] train episode 429: reward = 245.53, steps = 226\n",
      "11:40:10 [DEBUG] train episode 430: reward = 70.72, steps = 1000\n",
      "11:40:16 [DEBUG] train episode 431: reward = 69.00, steps = 149\n",
      "11:40:21 [DEBUG] train episode 432: reward = -17.51, steps = 156\n",
      "11:40:31 [DEBUG] train episode 433: reward = 232.31, steps = 251\n",
      "11:40:48 [DEBUG] train episode 434: reward = 256.14, steps = 439\n",
      "11:41:05 [DEBUG] train episode 435: reward = 270.95, steps = 450\n",
      "11:41:25 [DEBUG] train episode 436: reward = 278.93, steps = 513\n",
      "11:41:31 [DEBUG] train episode 437: reward = -26.74, steps = 164\n",
      "11:41:35 [DEBUG] train episode 438: reward = -0.10, steps = 101\n",
      "11:42:13 [DEBUG] train episode 439: reward = 129.38, steps = 1000\n",
      "11:42:53 [DEBUG] train episode 440: reward = 105.27, steps = 1000\n",
      "11:43:05 [DEBUG] train episode 441: reward = 210.13, steps = 303\n",
      "11:43:13 [DEBUG] train episode 442: reward = 223.78, steps = 220\n",
      "11:43:19 [DEBUG] train episode 443: reward = 65.32, steps = 174\n",
      "11:43:32 [DEBUG] train episode 444: reward = 243.66, steps = 340\n",
      "11:44:12 [DEBUG] train episode 445: reward = 51.24, steps = 1000\n",
      "11:44:21 [DEBUG] train episode 446: reward = 245.63, steps = 232\n",
      "11:45:00 [DEBUG] train episode 447: reward = 137.79, steps = 1000\n",
      "11:45:39 [DEBUG] train episode 448: reward = 113.79, steps = 1000\n",
      "11:45:56 [DEBUG] train episode 449: reward = 229.94, steps = 458\n",
      "11:46:35 [DEBUG] train episode 450: reward = 137.03, steps = 1000\n",
      "11:46:46 [DEBUG] train episode 451: reward = 267.52, steps = 298\n",
      "11:47:23 [DEBUG] train episode 452: reward = 288.69, steps = 950\n",
      "11:48:02 [DEBUG] train episode 453: reward = 145.29, steps = 1000\n",
      "11:48:40 [DEBUG] train episode 454: reward = 160.21, steps = 1000\n",
      "11:49:19 [DEBUG] train episode 455: reward = 130.20, steps = 1000\n",
      "11:49:59 [DEBUG] train episode 456: reward = 120.77, steps = 1000\n",
      "11:50:37 [DEBUG] train episode 457: reward = 126.23, steps = 1000\n",
      "11:51:08 [DEBUG] train episode 458: reward = 215.63, steps = 802\n",
      "11:51:46 [DEBUG] train episode 459: reward = 101.99, steps = 1000\n",
      "11:52:25 [DEBUG] train episode 460: reward = 131.59, steps = 1000\n",
      "11:52:33 [DEBUG] train episode 461: reward = 244.61, steps = 205\n",
      "11:52:44 [DEBUG] train episode 462: reward = 250.38, steps = 310\n",
      "11:53:23 [DEBUG] train episode 463: reward = 108.69, steps = 1000\n",
      "11:53:31 [DEBUG] train episode 464: reward = 255.06, steps = 211\n",
      "11:54:10 [DEBUG] train episode 465: reward = 139.20, steps = 1000\n",
      "11:54:48 [DEBUG] train episode 466: reward = 120.25, steps = 1000\n",
      "11:55:27 [DEBUG] train episode 467: reward = 124.46, steps = 1000\n",
      "11:56:06 [DEBUG] train episode 468: reward = 147.71, steps = 1000\n",
      "11:56:19 [DEBUG] train episode 469: reward = 207.01, steps = 339\n",
      "11:56:58 [DEBUG] train episode 470: reward = 145.63, steps = 1000\n",
      "11:57:37 [DEBUG] train episode 471: reward = 136.65, steps = 1000\n",
      "11:57:50 [DEBUG] train episode 472: reward = 246.61, steps = 323\n",
      "11:58:28 [DEBUG] train episode 473: reward = 151.00, steps = 1000\n",
      "11:58:50 [DEBUG] train episode 474: reward = 238.25, steps = 562\n",
      "11:58:52 [DEBUG] train episode 475: reward = -34.60, steps = 76\n",
      "11:59:30 [DEBUG] train episode 476: reward = 136.21, steps = 1000\n",
      "12:00:09 [DEBUG] train episode 477: reward = 142.19, steps = 1000\n",
      "12:00:47 [DEBUG] train episode 478: reward = 125.15, steps = 1000\n",
      "12:01:25 [DEBUG] train episode 479: reward = 133.39, steps = 1000\n",
      "12:02:03 [DEBUG] train episode 480: reward = 142.66, steps = 1000\n",
      "12:02:42 [DEBUG] train episode 481: reward = 161.41, steps = 1000\n",
      "12:03:20 [DEBUG] train episode 482: reward = 138.70, steps = 1000\n",
      "12:03:58 [DEBUG] train episode 483: reward = 149.07, steps = 1000\n",
      "12:04:36 [DEBUG] train episode 484: reward = 117.19, steps = 1000\n",
      "12:04:40 [DEBUG] train episode 485: reward = 6.27, steps = 104\n",
      "12:05:18 [DEBUG] train episode 486: reward = 112.67, steps = 1000\n",
      "12:05:23 [DEBUG] train episode 487: reward = 224.83, steps = 149\n",
      "12:05:29 [DEBUG] train episode 488: reward = 67.36, steps = 139\n",
      "12:06:01 [DEBUG] train episode 489: reward = 158.32, steps = 834\n",
      "12:06:39 [DEBUG] train episode 490: reward = 178.29, steps = 1000\n",
      "12:06:47 [DEBUG] train episode 491: reward = 224.99, steps = 224\n",
      "12:07:25 [DEBUG] train episode 492: reward = 135.58, steps = 1000\n",
      "12:08:04 [DEBUG] train episode 493: reward = 171.78, steps = 1000\n",
      "12:08:42 [DEBUG] train episode 494: reward = 141.74, steps = 1000\n",
      "12:09:21 [DEBUG] train episode 495: reward = 143.79, steps = 1000\n",
      "12:09:27 [DEBUG] train episode 496: reward = 1.35, steps = 147\n",
      "12:09:35 [DEBUG] train episode 497: reward = -30.79, steps = 211\n",
      "12:10:15 [DEBUG] train episode 498: reward = 143.50, steps = 1000\n",
      "12:10:53 [DEBUG] train episode 499: reward = 140.86, steps = 1000\n",
      "12:11:32 [DEBUG] train episode 500: reward = 126.08, steps = 1000\n",
      "12:12:11 [DEBUG] train episode 501: reward = 128.06, steps = 1000\n",
      "12:12:21 [DEBUG] train episode 502: reward = 201.06, steps = 260\n",
      "12:12:59 [DEBUG] train episode 503: reward = 144.96, steps = 1000\n",
      "12:13:15 [DEBUG] train episode 504: reward = 214.63, steps = 416\n",
      "12:13:22 [DEBUG] train episode 505: reward = 6.61, steps = 179\n",
      "12:14:00 [DEBUG] train episode 506: reward = 89.71, steps = 1000\n",
      "12:14:39 [DEBUG] train episode 507: reward = 202.74, steps = 986\n",
      "12:14:44 [DEBUG] train episode 508: reward = -1.13, steps = 135\n",
      "12:15:24 [DEBUG] train episode 509: reward = 118.31, steps = 1000\n",
      "12:16:03 [DEBUG] train episode 510: reward = 108.18, steps = 1000\n",
      "12:16:15 [DEBUG] train episode 511: reward = 209.12, steps = 315\n",
      "12:16:54 [DEBUG] train episode 512: reward = 136.71, steps = 1000\n",
      "12:17:33 [DEBUG] train episode 513: reward = 113.80, steps = 1000\n",
      "12:18:12 [DEBUG] train episode 514: reward = 168.73, steps = 1000\n",
      "12:18:34 [DEBUG] train episode 515: reward = 211.40, steps = 554\n",
      "12:18:38 [DEBUG] train episode 516: reward = -33.62, steps = 105\n",
      "12:19:17 [DEBUG] train episode 517: reward = 116.70, steps = 1000\n",
      "12:19:57 [DEBUG] train episode 518: reward = 119.32, steps = 1000\n",
      "12:20:36 [DEBUG] train episode 519: reward = 92.17, steps = 1000\n",
      "12:20:45 [DEBUG] train episode 520: reward = 199.84, steps = 225\n",
      "12:21:01 [DEBUG] train episode 521: reward = 186.73, steps = 430\n",
      "12:21:40 [DEBUG] train episode 522: reward = 123.72, steps = 1000\n",
      "12:22:20 [DEBUG] train episode 523: reward = 114.84, steps = 1000\n",
      "12:23:00 [DEBUG] train episode 524: reward = 134.94, steps = 1000\n",
      "12:23:22 [DEBUG] train episode 525: reward = 237.49, steps = 579\n",
      "12:24:01 [DEBUG] train episode 526: reward = 113.79, steps = 1000\n",
      "12:24:07 [DEBUG] train episode 527: reward = 3.46, steps = 170\n",
      "12:24:11 [DEBUG] train episode 528: reward = 19.38, steps = 106\n",
      "12:24:17 [DEBUG] train episode 529: reward = 50.39, steps = 151\n",
      "12:24:22 [DEBUG] train episode 530: reward = -35.07, steps = 124\n",
      "12:24:36 [DEBUG] train episode 531: reward = 203.90, steps = 360\n",
      "12:24:45 [DEBUG] train episode 532: reward = -87.35, steps = 239\n",
      "12:24:55 [DEBUG] train episode 533: reward = 230.81, steps = 262\n",
      "12:25:06 [DEBUG] train episode 534: reward = 191.13, steps = 293\n",
      "12:25:13 [DEBUG] train episode 535: reward = -53.85, steps = 182\n",
      "12:25:23 [DEBUG] train episode 536: reward = -101.43, steps = 253\n",
      "12:25:37 [DEBUG] train episode 537: reward = 175.83, steps = 353\n",
      "12:25:48 [DEBUG] train episode 538: reward = 218.88, steps = 288\n",
      "12:25:53 [DEBUG] train episode 539: reward = -22.08, steps = 132\n",
      "12:26:00 [DEBUG] train episode 540: reward = -40.94, steps = 201\n",
      "12:26:09 [DEBUG] train episode 541: reward = -17.87, steps = 214\n",
      "12:26:48 [DEBUG] train episode 542: reward = 158.65, steps = 1000\n",
      "12:27:28 [DEBUG] train episode 543: reward = 115.29, steps = 1000\n",
      "12:27:45 [DEBUG] train episode 544: reward = 162.96, steps = 430\n",
      "12:27:56 [DEBUG] train episode 545: reward = 224.67, steps = 281\n",
      "12:28:28 [DEBUG] train episode 546: reward = 203.53, steps = 808\n",
      "12:28:37 [DEBUG] train episode 547: reward = 4.79, steps = 238\n",
      "12:29:17 [DEBUG] train episode 548: reward = 129.84, steps = 1000\n",
      "12:29:26 [DEBUG] train episode 549: reward = 235.79, steps = 248\n",
      "12:29:38 [DEBUG] train episode 550: reward = 231.07, steps = 288\n",
      "12:29:47 [DEBUG] train episode 551: reward = 242.58, steps = 256\n",
      "12:29:53 [DEBUG] train episode 552: reward = 26.45, steps = 137\n",
      "12:30:04 [DEBUG] train episode 553: reward = 268.18, steps = 287\n",
      "12:30:20 [DEBUG] train episode 554: reward = 210.02, steps = 416\n",
      "12:30:28 [DEBUG] train episode 555: reward = 241.36, steps = 208\n",
      "12:30:41 [DEBUG] train episode 556: reward = 239.09, steps = 314\n",
      "12:31:06 [DEBUG] train episode 557: reward = 215.17, steps = 631\n",
      "12:31:46 [DEBUG] train episode 558: reward = 133.01, steps = 1000\n",
      "12:31:58 [DEBUG] train episode 559: reward = 205.48, steps = 309\n",
      "12:32:04 [DEBUG] train episode 560: reward = -5.08, steps = 161\n",
      "12:32:09 [DEBUG] train episode 561: reward = 5.65, steps = 133\n",
      "12:32:49 [DEBUG] train episode 562: reward = 138.11, steps = 1000\n",
      "12:33:28 [DEBUG] train episode 563: reward = 109.73, steps = 1000\n",
      "12:34:06 [DEBUG] train episode 564: reward = 146.65, steps = 1000\n",
      "12:34:45 [DEBUG] train episode 565: reward = 173.19, steps = 1000\n",
      "12:34:49 [DEBUG] train episode 566: reward = 5.80, steps = 122\n",
      "12:34:53 [DEBUG] train episode 567: reward = -52.19, steps = 103\n",
      "12:35:12 [DEBUG] train episode 568: reward = 168.04, steps = 490\n",
      "12:35:51 [DEBUG] train episode 569: reward = 144.85, steps = 1000\n",
      "12:35:55 [DEBUG] train episode 570: reward = 5.84, steps = 125\n",
      "12:36:33 [DEBUG] train episode 571: reward = 83.16, steps = 1000\n",
      "12:36:39 [DEBUG] train episode 572: reward = 3.79, steps = 158\n",
      "12:36:47 [DEBUG] train episode 573: reward = -29.89, steps = 210\n",
      "12:37:25 [DEBUG] train episode 574: reward = 155.09, steps = 1000\n",
      "12:37:32 [DEBUG] train episode 575: reward = 261.25, steps = 182\n",
      "12:37:38 [DEBUG] train episode 576: reward = 33.45, steps = 163\n",
      "12:37:49 [DEBUG] train episode 577: reward = 251.77, steps = 290\n",
      "12:37:57 [DEBUG] train episode 578: reward = 202.44, steps = 210\n",
      "12:38:36 [DEBUG] train episode 579: reward = 140.15, steps = 1000\n",
      "12:39:14 [DEBUG] train episode 580: reward = 116.79, steps = 1000\n",
      "12:39:22 [DEBUG] train episode 581: reward = -28.87, steps = 209\n",
      "12:39:34 [DEBUG] train episode 582: reward = 230.60, steps = 319\n",
      "12:39:42 [DEBUG] train episode 583: reward = 15.11, steps = 223\n",
      "12:40:21 [DEBUG] train episode 584: reward = 109.42, steps = 1000\n",
      "12:41:00 [DEBUG] train episode 585: reward = 119.07, steps = 1000\n",
      "12:41:05 [DEBUG] train episode 586: reward = 39.63, steps = 139\n",
      "12:41:12 [DEBUG] train episode 587: reward = -47.84, steps = 183\n",
      "12:41:36 [DEBUG] train episode 588: reward = 233.27, steps = 611\n",
      "12:41:46 [DEBUG] train episode 589: reward = 288.73, steps = 270\n",
      "12:42:25 [DEBUG] train episode 590: reward = 132.62, steps = 1000\n",
      "12:42:33 [DEBUG] train episode 591: reward = 251.68, steps = 228\n",
      "12:43:12 [DEBUG] train episode 592: reward = 117.34, steps = 1000\n",
      "12:43:17 [DEBUG] train episode 593: reward = 18.96, steps = 134\n",
      "12:43:30 [DEBUG] train episode 594: reward = 255.76, steps = 357\n",
      "12:44:08 [DEBUG] train episode 595: reward = 135.59, steps = 1000\n",
      "12:44:46 [DEBUG] train episode 596: reward = 146.12, steps = 1000\n",
      "12:45:24 [DEBUG] train episode 597: reward = 163.82, steps = 1000\n",
      "12:46:02 [DEBUG] train episode 598: reward = 121.32, steps = 1000\n",
      "12:46:40 [DEBUG] train episode 599: reward = 146.93, steps = 1000\n",
      "12:47:18 [DEBUG] train episode 600: reward = 155.23, steps = 1000\n",
      "12:47:56 [DEBUG] train episode 601: reward = 158.77, steps = 1000\n",
      "12:48:34 [DEBUG] train episode 602: reward = 135.33, steps = 1000\n",
      "12:48:42 [DEBUG] train episode 603: reward = 296.22, steps = 224\n",
      "12:49:20 [DEBUG] train episode 604: reward = 118.72, steps = 1000\n",
      "12:49:57 [DEBUG] train episode 605: reward = 240.88, steps = 986\n",
      "12:50:05 [DEBUG] train episode 606: reward = 286.03, steps = 207\n",
      "12:50:11 [DEBUG] train episode 607: reward = -76.28, steps = 161\n",
      "12:50:20 [DEBUG] train episode 608: reward = 280.56, steps = 233\n",
      "12:50:31 [DEBUG] train episode 609: reward = 259.10, steps = 311\n",
      "12:50:40 [DEBUG] train episode 610: reward = 270.23, steps = 235\n",
      "12:50:50 [DEBUG] train episode 611: reward = 276.68, steps = 251\n",
      "12:50:56 [DEBUG] train episode 612: reward = 248.49, steps = 176\n",
      "12:51:08 [DEBUG] train episode 613: reward = 292.11, steps = 305\n",
      "12:51:32 [DEBUG] train episode 614: reward = 266.76, steps = 633\n",
      "12:51:38 [DEBUG] train episode 615: reward = 249.82, steps = 171\n",
      "12:51:45 [DEBUG] train episode 616: reward = 262.73, steps = 176\n",
      "12:51:56 [DEBUG] train episode 617: reward = 247.55, steps = 308\n",
      "12:51:57 [INFO] ==== test ====\n",
      "12:51:57 [DEBUG] test episode 0: reward = 261.92, steps = 243\n",
      "12:51:57 [DEBUG] test episode 1: reward = 254.51, steps = 231\n",
      "12:51:59 [DEBUG] test episode 2: reward = 247.75, steps = 847\n",
      "12:51:59 [DEBUG] test episode 3: reward = -4.16, steps = 326\n",
      "12:52:00 [DEBUG] test episode 4: reward = 225.08, steps = 267\n",
      "12:52:00 [DEBUG] test episode 5: reward = 282.01, steps = 256\n",
      "12:52:00 [DEBUG] test episode 6: reward = 256.62, steps = 231\n",
      "12:52:00 [DEBUG] test episode 7: reward = 269.03, steps = 205\n",
      "12:52:00 [DEBUG] test episode 8: reward = 288.36, steps = 218\n",
      "12:52:01 [DEBUG] test episode 9: reward = 24.78, steps = 265\n",
      "12:52:01 [DEBUG] test episode 10: reward = 239.47, steps = 221\n",
      "12:52:01 [DEBUG] test episode 11: reward = 1.40, steps = 238\n",
      "12:52:01 [DEBUG] test episode 12: reward = 274.42, steps = 208\n",
      "12:52:02 [DEBUG] test episode 13: reward = 261.07, steps = 299\n",
      "12:52:02 [DEBUG] test episode 14: reward = 253.14, steps = 218\n",
      "12:52:02 [DEBUG] test episode 15: reward = 288.40, steps = 234\n",
      "12:52:03 [DEBUG] test episode 16: reward = -130.06, steps = 650\n",
      "12:52:05 [DEBUG] test episode 17: reward = 166.18, steps = 1000\n",
      "12:52:05 [DEBUG] test episode 18: reward = 265.44, steps = 195\n",
      "12:52:05 [DEBUG] test episode 19: reward = 271.89, steps = 238\n",
      "12:52:05 [DEBUG] test episode 20: reward = 278.74, steps = 245\n",
      "12:52:05 [DEBUG] test episode 21: reward = 271.36, steps = 233\n",
      "12:52:06 [DEBUG] test episode 22: reward = -12.77, steps = 178\n",
      "12:52:06 [DEBUG] test episode 23: reward = 247.23, steps = 218\n",
      "12:52:06 [DEBUG] test episode 24: reward = 201.95, steps = 434\n",
      "12:52:07 [DEBUG] test episode 25: reward = 245.91, steps = 195\n",
      "12:52:07 [DEBUG] test episode 26: reward = 243.44, steps = 195\n",
      "12:52:07 [DEBUG] test episode 27: reward = 274.77, steps = 226\n",
      "12:52:07 [DEBUG] test episode 28: reward = 2.28, steps = 227\n",
      "12:52:09 [DEBUG] test episode 29: reward = 94.40, steps = 1000\n",
      "12:52:09 [DEBUG] test episode 30: reward = 286.08, steps = 301\n",
      "12:52:09 [DEBUG] test episode 31: reward = 259.12, steps = 249\n",
      "12:52:10 [DEBUG] test episode 32: reward = 272.56, steps = 232\n",
      "12:52:10 [DEBUG] test episode 33: reward = 256.48, steps = 219\n",
      "12:52:10 [DEBUG] test episode 34: reward = 276.92, steps = 242\n",
      "12:52:10 [DEBUG] test episode 35: reward = 250.72, steps = 250\n",
      "12:52:11 [DEBUG] test episode 36: reward = 274.10, steps = 180\n",
      "12:52:12 [DEBUG] test episode 37: reward = 135.72, steps = 1000\n",
      "12:52:12 [DEBUG] test episode 38: reward = 276.21, steps = 228\n",
      "12:52:13 [DEBUG] test episode 39: reward = 271.82, steps = 261\n",
      "12:52:13 [DEBUG] test episode 40: reward = 272.34, steps = 231\n",
      "12:52:13 [DEBUG] test episode 41: reward = 278.68, steps = 315\n",
      "12:52:13 [DEBUG] test episode 42: reward = 289.59, steps = 229\n",
      "12:52:14 [DEBUG] test episode 43: reward = 19.72, steps = 197\n",
      "12:52:14 [DEBUG] test episode 44: reward = 269.59, steps = 216\n",
      "12:52:15 [DEBUG] test episode 45: reward = 230.64, steps = 939\n",
      "12:52:15 [DEBUG] test episode 46: reward = 265.88, steps = 262\n",
      "12:52:16 [DEBUG] test episode 47: reward = 273.02, steps = 245\n",
      "12:52:16 [DEBUG] test episode 48: reward = 279.87, steps = 193\n",
      "12:52:16 [DEBUG] test episode 49: reward = 256.61, steps = 267\n",
      "12:52:16 [DEBUG] test episode 50: reward = 8.28, steps = 224\n",
      "12:52:17 [DEBUG] test episode 51: reward = 282.33, steps = 216\n",
      "12:52:18 [DEBUG] test episode 52: reward = 134.88, steps = 1000\n",
      "12:52:18 [DEBUG] test episode 53: reward = 286.73, steps = 241\n",
      "12:52:19 [DEBUG] test episode 54: reward = 280.60, steps = 222\n",
      "12:52:19 [DEBUG] test episode 55: reward = 250.07, steps = 233\n",
      "12:52:19 [DEBUG] test episode 56: reward = 258.26, steps = 284\n",
      "12:52:19 [DEBUG] test episode 57: reward = 258.39, steps = 225\n",
      "12:52:20 [DEBUG] test episode 58: reward = 252.15, steps = 230\n",
      "12:52:21 [DEBUG] test episode 59: reward = 177.74, steps = 754\n",
      "12:52:21 [DEBUG] test episode 60: reward = 250.30, steps = 275\n",
      "12:52:21 [DEBUG] test episode 61: reward = 252.47, steps = 178\n",
      "12:52:21 [DEBUG] test episode 62: reward = 237.86, steps = 198\n",
      "12:52:21 [DEBUG] test episode 63: reward = 276.02, steps = 226\n",
      "12:52:21 [DEBUG] test episode 64: reward = 247.22, steps = 203\n",
      "12:52:22 [DEBUG] test episode 65: reward = -30.39, steps = 412\n",
      "12:52:22 [DEBUG] test episode 66: reward = 265.27, steps = 279\n",
      "12:52:22 [DEBUG] test episode 67: reward = 280.11, steps = 200\n",
      "12:52:23 [DEBUG] test episode 68: reward = 259.68, steps = 250\n",
      "12:52:23 [DEBUG] test episode 69: reward = 219.61, steps = 288\n",
      "12:52:23 [DEBUG] test episode 70: reward = 240.97, steps = 228\n",
      "12:52:23 [DEBUG] test episode 71: reward = 263.67, steps = 254\n",
      "12:52:24 [DEBUG] test episode 72: reward = 280.41, steps = 219\n",
      "12:52:24 [DEBUG] test episode 73: reward = 270.40, steps = 163\n",
      "12:52:24 [DEBUG] test episode 74: reward = 234.56, steps = 190\n",
      "12:52:24 [DEBUG] test episode 75: reward = 278.69, steps = 245\n",
      "12:52:24 [DEBUG] test episode 76: reward = 246.37, steps = 277\n",
      "12:52:25 [DEBUG] test episode 77: reward = 36.31, steps = 283\n",
      "12:52:25 [DEBUG] test episode 78: reward = 250.18, steps = 378\n",
      "12:52:25 [DEBUG] test episode 79: reward = 236.87, steps = 241\n",
      "12:52:26 [DEBUG] test episode 80: reward = 284.36, steps = 261\n",
      "12:52:26 [DEBUG] test episode 81: reward = 264.29, steps = 250\n",
      "12:52:26 [DEBUG] test episode 82: reward = 269.65, steps = 232\n",
      "12:52:26 [DEBUG] test episode 83: reward = 275.08, steps = 248\n",
      "12:52:27 [DEBUG] test episode 84: reward = 261.39, steps = 245\n",
      "12:52:27 [DEBUG] test episode 85: reward = 249.30, steps = 182\n",
      "12:52:27 [DEBUG] test episode 86: reward = 222.31, steps = 350\n",
      "12:52:27 [DEBUG] test episode 87: reward = 259.59, steps = 171\n",
      "12:52:28 [DEBUG] test episode 88: reward = 251.40, steps = 180\n",
      "12:52:28 [DEBUG] test episode 89: reward = 244.87, steps = 193\n",
      "12:52:29 [DEBUG] test episode 90: reward = 149.38, steps = 1000\n",
      "12:52:29 [DEBUG] test episode 91: reward = 270.59, steps = 274\n",
      "12:52:29 [DEBUG] test episode 92: reward = 6.33, steps = 182\n",
      "12:52:30 [DEBUG] test episode 93: reward = -13.06, steps = 269\n",
      "12:52:30 [DEBUG] test episode 94: reward = 256.70, steps = 204\n",
      "12:52:30 [DEBUG] test episode 95: reward = 267.60, steps = 262\n",
      "12:52:31 [DEBUG] test episode 96: reward = 261.67, steps = 850\n",
      "12:52:31 [DEBUG] test episode 97: reward = 297.19, steps = 229\n",
      "12:52:32 [DEBUG] test episode 98: reward = -91.23, steps = 204\n",
      "12:52:32 [DEBUG] test episode 99: reward = 252.15, steps = 185\n",
      "12:52:32 [INFO] average episode reward = 218.66 ± 97.16\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABR3ElEQVR4nO2dd5xcVdnHf8/MbMum97KpkIQUIJAQOtITacFX1KAoYoklvPZCURQVxK68CooiRUGMgIBSQxchhE1IJQnpyZK26ZvNZndn5rx/3HvunHvvOffeKVtm5/l+PvvZmXPbuXdmnvOc33nOc0gIAYZhGKa0iHV0BRiGYZj2h40/wzBMCcLGn2EYpgRh488wDFOCsPFnGIYpQRIdXYGo9O/fX4waNaqjq8EwDFNULFq0aLcQYoC3vGiM/6hRo1BbW9vR1WAYhikqiGizrpxlH4ZhmBKEjT/DMEwJwsafYRimBGHjzzAMU4LkbfyJqJKIFhLRUiJaSUQ32+V9iWg+Ea21//dRjrmeiNYR0RoimpFvHRiGYZjsKITn3wzgXCHE8QCmAJhJRKcAuA7AC0KIsQBesN+DiCYCmA1gEoCZAO4gongB6sEwDMNEJG/jLywO2W/L7D8BYBaA++zy+wBcbr+eBeAhIUSzEGIjgHUApudbD4ZhGCY6BdH8iShOREsA7AIwXwjxJoBBQojtAGD/H2jvPgzAVuXwOrtMd945RFRLRLX19fWFqCrDMEyn4lBzEo+9/V67X7cgxl8IkRJCTAFQA2A6EU0O2J10pzCc9y4hxDQhxLQBA3wT1BiGYYqeG/+5HF/5+xIsq9vvlB1qTmLBhj1tet2CRvsIIfYDeBmWlr+TiIYAgP1/l71bHYDhymE1ALYVsh4Mw3Q8b6zfg3W7DoXvWAK0ptJ4aOEWpNN+P3f7/iMAgMMtKafs6/OWYPZdC1Df0NxmdSpEtM8AIuptv64CcD6A1QCeAHC1vdvVAB63Xz8BYDYRVRDRaABjASzMtx4Mw3QurvzjApz/y1c6uhqdgjteWo/rHl2Ox5ea5Z0fP70az6zYAQB4d6fVaB5oam2zOhXC8x8C4CUiWgbgLVia/78B3AbgAiJaC+AC+z2EECsBzAPwDoBnAMwVQqS0Z2YYpijh5WHd7GywvPtDzWZTt3Trfnz+r4uw/3ALyuOWaT54pO2Mf96J3YQQywCcoCnfA+A8wzG3ALgl32szDNNxfPvhZRjQowLfmDHet+1gU7IDatR5aU2mAQDlcf+Qp/AMeb7vZy+jyZaA9h9uabM68QxfhikR7n5tI0Zd96RWd86WdFrg77Vb8duX1mm3725sO626GEnazzwRCze5B5pa0ZKyGot9jZ1b9mEYpoM5eKQVR1qD1dMfP7UKAJAqgCSzvj54IHe3PVBZVcbzNwE4xrws4Te5pA2AtNjfyTV/hmE6mOO+/xwu+b/XIu0bxfa/vWUf/r3MHIQno3i6leuN+55GS67o3a0sUp06K83JFEZd9yT+8Mr6vM6TTEWXfVRY9mEYJpSwsEppYtIRrP8H7ngd1z74tnF73b4mAEBNnyrtdqlZVxa553+k1TLaJnnLiyk0szUVXfZR+d1L67B2Z0NWx0SFjT/DMIHsbWzBtv1NrrK6fYcBAN0r9DEjqQKMK3QGYraj3mp77kEs3bofJ93yPB5ZVOfbJo/P9qmkBfA/d7ye5VHRYOPPMCWCFByieP4qJ/5wPk677UVXmfT8TTZejivo1Oynlm/HhpAxg86CvL+WZLjxX7X9IADgzY3+mbnS+KfSAn9dsBlPL9/ubAv7OBqa2yZyio0/w3RBhBBoNBiNQjjl9YcsecN0qqThIht3N+KLDyzGub8ojslfcr5ClGcm95lXW+ebnCVln1ufWoXvPLYCX3hgsfE8Hzt5RG6VzRI2/gzTBfm/F9dh0veexb5G/4BhNp5/0iB37LXPa5rMldIc19SSwjk/f9lXvqH+EA63dM55Adk0lOpz/ckzq13b5HPcsvdw6Hl0EVJtMWmOjT/DFDk6w/D4EiuNwB5NvL3w2OXvP7ESo657UnvuzYqxWrfrEEZd9yQWbtzrNCrqpRds2INR1z2Jq/70JlKyXNF9/rPWn5lXCIFzf/EK5ty/SHv9jsY71hGE+jkc9vS6WlJm4+3dQgR895KJePCzJ+PD02rQv3s5DrWB9JP3DF+GYToWk8QCAC+u3oWKRBzD+3YzRvvc+/om13vViL2waqfzWubpeXjRVjTa0TzquWbftQAA8Nq63ThrXH9fXeQ4gYocGH5t3W7jPXQUDUdaXeGzf1u4BVdON0sy6iB3i6fnY+pB6YgR4dNnjAYAnDqmH4jM8wDygT1/hilydJEo0mDc+tRqnOfR18MEBBneCABL6w74tu9VZp2a1AidrdNFAKkNl1cqKTRffGARfhcxZBMA7vnvJtf76x9dHri/envNre4HECVayEGx9W1l+AE2/gxT9LQmg8259EKjRvs0NGeMuy4VhDrxyHSuVNp9TUDfQ1HL7n5tY2C98uWp5Tvws2fXOO/TaWFMdZFMpfHL+e/6ypuT5lnU6rNoTnqNf4Ds43mGQTN+Cwkbf4YpcrwSA6APsZSEGf8mJa+8bl+pP5ti/AHg58/5DadsEFxlilGMt6GXq2PMDU9hzl/0Yw0mKW3/YXO6BfVZtSTTuOg3/8G59gB3NgO2sXZ6DKz5M0yRk5WkgPC4ctXw6WyglG/iMXIMni6qyHstnffbqjQICY3V21B/CFv3NWFA9wps29+E8ycOCq58ljyvjGlIXlqzCyP7dtPuv7exBYN6Vmq3qR9DcyqNd+y4fyA7+aa92kA2/gxT5CQ1RlVnQKKmd1C1eZ0sIo9PxMgx7if8cL7+XMq1dJq/WhbX5L3xzgfYdNvFATXPHyEErrnnLVQbchaZGjnA7/lHvqbnPcs+DMNEQif7BBEWu642Jjr5Q9o46fkHSRph51LLdJ5/e/DvZdtw7PefRXMy5fROGlv02v7egERrwqX5u4/PJqVPez0GNv4MU+RkL/vojbUsV73xpEanlx5u3Pb890XQwUdd9yR+r8mMqWr+sRz0jgOHW/HRPy7A9gPR4/G9fP+Jd9BwJIn9h1sDB3SBMM8/89ob7ZOVN99Oug8bf4YpcrShngHGxuSoS+OlGnydTi/3ixFBIHjWaliCtzDNP4yHF9fh9fV78IdXNgCwJmVtP9CET96zEN/4x9JI52i210Eoi8d8UTpejrSatwfF+QfdmvfzaK/+TyEWcB9ORC8R0SoiWklEX7bL+xLRfCJaa//voxxzPRGtI6I1RDQj3zowTCkT1fMPC/WUxsvl+WvOrXr+aSGw55B51a4g4//2ln349sPLnPc6zT8MWb8y+9jTbnsRp/74Rby8ph4Pa7Jr6jhie/tCiFCt/s2Ne1zRUCou2cezsE42vZpcekC5UAjPPwng60KICQBOATCXiCYCuA7AC0KIsQBesN/D3jYbwCQAMwHcQUTFnfSbYTqQFjvOn1yTg8z7m+yxNOrJdLBOL22cHPANimEPWjXsyj8uQO3mfc77bHPdA8COg9bC6P9ZuxvPrdxhrofnPuRsZSBT/7Twx+dLbr/SWqb8+VW7cMM/9ZO91Hv1jcN0PtUnf+MvhNguhFhsv24AsArAMACzANxn73YfgMvt17MAPCSEaBZCbASwDsD0fOvBMJ2BSTc9g19pJgcVko27G518+kBGpikLMZ5h0T6yWDWUetnHKovZnr9uXECSTV7/eATZR40+WrJ1vzMLd/WOBm3M/sbdjbjwV684jYREt/CNEMKo+ZcpdVulhHC66qbcqrcHkY09LxrZR4WIRgE4AcCbAAYJIbYDVgMBYKC92zAAW5XD6uwy3fnmEFEtEdXW1/uTQjFMZ6OxJYXfvLC2Ta9xzs9fxhk/ecl5L2WfKMYTCEjJoPP8A2Qf6fkHGfhUwCxaL1E0f3WMYLXBCKvc9/omvLvzEP5RuzV037Qwh2iqcfqm56w2qvmkzY61U7hPwYw/EXUH8AiArwghgj4V3Z1pH5UQ4i4hxDQhxLQBAwYUopoM0+WQsk8iomZuivZJO9E+GQOok31cA75CBMs+aeEy2EFE0bqTWUYHyWUmN+8JT6WcFsIo+6j22NRIBTVyebQFbUZBjD8RlcEy/A8IIR61i3cS0RB7+xAAu+zyOgDDlcNrAJhXimYYJhDp+atGKWhGqclGyVTPqoHVDSbLxiMRt6J9dGkbMtcS2kloOqL0XNT6RNHG+3QrBwBs2tMYum9aCF+IpiQWyfM3nztoXp1vklexaP5kfcvuBrBKCPFLZdMTAK62X18N4HGlfDYRVRDRaABjASzMtx4M05W5+s8L8cl79D8TqbkHGc+DR5RkbaZoH12cv8Zwe9M7BHn+ybQwRiN5qxHN+Gfn+cu6RsnLLwTQktJr/upwSiKuN5tB8lc2C+i0V7RPIdI7nA7g4wCWE9ESu+wGALcBmEdEnwawBcCHAEAIsZKI5gF4B1ak0FwhRPDMCoYpcV551zzmlXG8zUbjI39YEBrqqY/20Wn+1v84EdIhmr9lUKPJPlE0f7U+UYKD5L0ExedLgjx/tSdlqmfQTOdsFuJqrwHfvI2/EOI1mOt7nuGYWwDcku+1GaYz0RZL7UVBGu0gh3HV9oOOZ22e5KWb4Rsc7WOFegYbVpNB9RLJ81fSV0eZNSsbi7CZu0BwqKfqjZs8/yDZR9fgCiGsRsWb0rlYZB+GYSwKsTB6bte1jX/E/Y3GX2r+IbKPUDx/IURoOGcUwwtEM/5L6/Y7r6NMbpP1j5JoLR0wycs74NucTGHug4uxoT4TMho0p0G3yfTcimmSF8PkhBACL6zaGTkUsLOTja5b2Ota/12TvAL3jx7tE2XAN2gZScAsuXiPimL8//dvbzuvo8hJ0vOP8hULivP3Dvgu2LAXTy7bjs/cV4sV71mrnaUCxj50zzzsubU1bPyZDuORxe/h0/fV4sGFWzq6KgWh44y/9PyjeYzZaf7BoZ5Ronmiev7Z5vZpjeDNZ2NgRYDsozasZXHCwSZrAH3D7kZnnd+ga+ke+cptVkS8P9qHPX+mi7PDzsSYT0bGzkTEcPY2uK5f888pvYNdf1WO0EkT3kleQTN8AeC9/UcCt0vi9gjusrr9uPe/G437yfuN4vkHeeO+80bU/OOxGBqOJH37BD0HXYP7wTtfd3oN7mtFqW3+8GIuDFMgCuX57z/cgl5VZZE9QEf2CdlPbg+b5BXmyfvTOwTv/yVFqglCjqNe9tv/AgCuPm2Udr9DLUn0rCyLpOO3ZuH5/2vpNvzWsMB7zBPt03DEnca6qSUV+BxMm/5RuxXL6twNQFGmd2CYUqYQxn/xln2Y8oP5eDYgSVmU6wav5BV8nrABXLk5ESNA6FNA5II3sdtBjXcNwJFcWkIaqUSMAiegeTEZfsA/4Ov1/PcdbgnpZei33ffGZl8Zyz4MU2QUYvzumRWW0V+70594zETKkX0Kk94hzJPP1vM3V8T91jvg+9ra3drDDjZZhjfM869IxAInqWWDK84/7vf89za25OT56+CVvBimyChE1JLMGDmin34BcR3ZdjjMnr/1P8xbdqV0RrhMZMKrkXvPMvfBxdrj5GzlsFDPirK41iBnu/IZ4DbIcY3nv7exJXCeR1a9wiKa4cswDAoj++w6aC2Mks2pZHy5O9QzaCUv/cnvfX0Tdh44gmNrekW6bjxPz189bEivSmyoP4RHF4cvwDL7rgU4vqYXDjSZl48EpOfvN/S5GX9V84/hULN7AZv9Ta1Zx/mbYM2fYYqMQsg+0sZkY1CjzPB1768vf/DNLXhh9a7IsohM7xCk+V91yohI5yqLx7By20F8bV60pReX1h3Apj2HnRW8TOfUe/7Zf1Cq8a/bdxivrnWn20im0lnP8I1yrbaEjT/T4XRQeHzBKWR6h2wkJHlZ1dsPDvW0ZuXO+t1/8dLqXb7tXoNp0qDj9oCvqbGoSMTQr7oiuPI2uazfCwC97aydJnSSlMnzD2pI1Of5/KpdvolryZB1Czqh6sPGn2EKRVC3PyrOalpZnCsz4BvxGgAONLVi6db9+Nq8JcbzSUwzb+WArykqZ8H156FnVVlgXe6+ehoemnOKzzOWyyZKqsri+NTpo3HWOPe6HkdazRPITJKUaaC4zJCzB7C88UE9zQ1ZOi0Cvfsgx+CbM8a73rPsw5QM7eXptDWFnK2fTUSKLrdPWHoH4UhF/j293rJp/CARIyTTAs+v2qnd3qe6HH2rg43/qP7VOGVMP99gby9Po1EWJ9x06UQM613pKtdNtpJYq4xF1/wDjX8MeOpLZxq3J9PBOY7SAjh5dF/n/c2XTXJezz3naPe1WPZhSoWuIvsUMkdRdsbf+h/1CCFE4MQwr8E02aIoRuqYwT0Dt/e1ZRv1O3DssF7oUemORZFLG3qvecXUGuO5hRDaSV4mzT/M8+/XvcLo/aeF2/PvqdRf2NumDO+NPt3K7PMZL9Vurj8bf4YpEIWI9pG2LSvjb+/rurzGMEtvP50OHiT2a/5mzz+Mowd2D9ze2zaGwm66ThnTF/d9arrLeKp1UCWoWz9wLH52xXHGczc0J/Hksu2+cpPnXx6g+cvLmp5FMiVcPb/+3TONxIurd0EIq5cljw+ak1FMi7kwTF6w7KM7V/ayjwjx/WX9rNW3pAH0P3xvw2P6fKJk4SyLx/Dj/zkWd768Hlv2ZtbRvWDiIHx0+gjHCMrbnTlpMPpWl/sMtLyUes1R/boFGlGTJGTKCWTK02+h73lI5CC6pF/3cmzYbS0d+en7au1jM88yRoR/fP5Upyfgv1Lbw54/0+F0GdmngDeSi+wTNZOBuvqWzpZ5k5uZDF4U4w8AV04fgcnD3PLP9y6diHOOGei8d9YIsM9ZkXCbJsfzV+oSyzFCKCwb6Oj+1b4yx/M3WMxUWrgGdbuV+/1qoozHHyPgpFF9cfTAHgCAFTfPwLhB3Z392oNCLeD+ZyLaRUQrlLK+RDSfiNba//so264nonVEtIaIZhSiDgzT0RQy1DObaB9tbp/AIzLr7ur2e29/E/pWZ0IoTcbIJPtcOX0E5n/1LFdZY3NwWmfvALS3YdHJPqbG56Mnj8CJI3q7ypbcdAF++eHjAZg1/2Qqjf7dK/CDWZN822IU7Pkn08L1menqFiNyGi/vebpXJJwxh2Ib8L0XwExP2XUAXhBCjAXwgv0eRDQRwGwAk+xj7iCieIHqwRQhXUX2KVB+M+tcBgOla2Ck5r/vcAvuf2NTaCMU5vlv3tOIs8cNwEg7xYTp4zF53ieN6oOxg3q4yppaLOP/1fPHYdrIPhjUs1J3qGM0vYOvGc+bfGVePnPGaEwe5p6l3LtbOXpUWhKLSfNvTQskYuTqXWSuFSL7pIWr56Xbj4ic+wv6zheV5y+EeBXAXk/xLAD32a/vA3C5Uv6QEKJZCLERwDoA0wtRD6Y4YdnHj8nz1xVL2edwSwo3Pb4Sy+oOhE7ykgZQF8a582AzhvfthrPGWjH1JiOvM5ImEvZg6vkTB+LhL5zmM+7ytuIGz1/2CBIu4++//nnHDMSYAd212+SxJs0/lRaIx0h7v/J08v/UkX1cvaOkJ85f1yuKUUY2ai/vPoi21PwHCSG2A4D9Xwp8wwBsVfars8t8ENEcIqolotr6+nrdLkwR016pa9uLQhj/tBORkznXc0p656AF1SVhklFaZM5j+ggG96rMeKmG86hG8vSj+wVe85cfnoJvzhiPiUP0oZ+yyvKcXuMp6+JdTtGL/E7p7kvub5rk1ZpKIxbTG2ZZL6dxUiJ3AP+Ar7ZuUGSfAMtbbLJPNujuTPttFULcJYSYJoSYNmDAAN0uTBFTSI28M1CI25EGRDXgc/6yyHmtWy3Ka/zLYrFAzd9adF16/nrK4zGtwVVx6+/BpmRwr0rMPedoY4Mv70F2CLz76aJ99NKKeZtsUEyyTzIlECPSykneUM94jKB2Xryev673EKNw+Ui9h7amLY3/TiIaAgD2f5lEpA7AcGW/GgDb2rAeDNMu5JIn3os8hSmxm26wMmo6hsw1lAFfUwx/nBxjqdsnRu6Go6xASehNRlE34Kv10Mn9H8jMJZChnCbjn0oLy6M3DNYCGcMc94wNWOkdMvvrpgzEFEkpqNcbdS3mfGlL4/8EgKvt11cDeFwpn01EFUQ0GsBYAAvbsB5MiZJKC8x9cDGWe5bJaysKIftIQ26aLazLoOndVWf81TKhDPi+t1+/fnLcZaj822Me2SPXsEunTsp5dahGV62jrl7q/xmTBuE/3zrHtb9pBbDWdNoy0BF6FPEYuQx4yuv5m3pLSqiniaJaw5eI/gbgbAD9iagOwPcA3AZgHhF9GsAWAB8CACHESiKaB+AdAEkAc4UQwXFgDJMDdfsO48ll27G87gBetQ1AW1KISV6O7GOIHNL1CLzyma4RqkzE0GhH3KRFeE77RCzj+etlEHI1CkEZMaPgjfP3IstVb1s3JyszMGu96Ftd4UT5OLKPQfMXwrrXoGgfWY9EjFx1/dNrGz310J/DlKZCdw9tTUGMvxDiSsOm8wz73wLglkJcmyl+usq4byHGMKTh/vN/N+KMsf1w7jGDXNt1Rtsr++jkp3KX8Q9OQgZYGn5mwFfvCbuWNgzR/MOQzy4b2UdnYMnjWSc0PYWghs/bqHmvr4acBslr+nNkGqxg7774ZR+GiURXGfcthOafckX5+LNl6jR/72XTQvgMY7kyY1Yomr8JNd69W4V/Gk6M3E1Crvn4/efVl+smf0WJx1f3l+GmQcY/bpB95GnUkNOgW9ZtIqVXEZzbx3zeQsLGn2EKRCFkH1Wy0adb9hsub49D1whVJDIGXJ3kZSIeI8RlbP6EQfjR5ZNd263IFff++eDE+ZvWDlA87qBryhJd1E8mzt987zHSe/TeHkWY569vQEg5T1CvgT1/pkRoq++6tIlhCc8Kd73Cev46TVsb7aPR/L2PdLAyo9Za5CRc85deqhACV50y0rW9sSXlmW2br+Zvyz4hmr9rkpchnBJQewrqOYKjfeQ5g0I91Xpkq9urM3yz7TW0BWz8mQ7DMc5tZJsLOeM22vXyP4c6oKszLvo4f/d7r2175Ztno1qRboTQL2+oosofpvtSazdmgD8ZWjbIa+iknO4VCfxw1mTfdtNAtLpN5/kHJXaLk6lRccs1JnlIoh2PAIy5fVzXaierzMaf6TAKmQJZh5O/pp18qUIs45gOMf6q5+/k59fIPuqhMSKf5h/q+cczA5+mRlQ1cJ89c4yrd5Et8l50UsqjXzwNxw/vDcAj+2jda+uf8+w00lRumr+7UYkb5CGnGobeQyxm3u4cywO+TBjptMCtT63Clj2Hw3fuhEhj2VZtgPSi20v2KUicf0isuKr5y129cwK88hORO1Gamt7BhBrtI0/310+fjG9cOM51XqeuMcKpRwWneAhC1iYoLQMQRfYJ9/yDNH8yzPD1jiEk4vrJYJl6aMpiamK3jtd92PgXMWt2NuCuVzfgiw8uCt+5E2LyXAtFIaJvsqEgoZ6KUxqm+adNnr8QLu8xRoRyl/EXobJPIpY5gzz/GWP74+LjhrrOWzACZJ8yRQcJG2fwGl31rTS8jc3mdX+9OXu811JTXgRNbdCGxxrqbLpWW8PGv4iRv/mwH3JUbv7XSjz45paCnCsKzgpUbWSj29v4FyKlc5jn36q0DpmVubz18Kd78Hr+YZO81Bm+6uejTuby1u6CidacBG865SgERfvElWu6JnlpdXV3ndVd5FyEJ5aas8mYsnrqQj2zlX3UZRyD1nturwFfXsaxiJFfsEIZz3v+uwmAtRhGeyDtTzYLn+851IznV+3ER04Kr2OynTX/QqZ3AEyhnhrP3/P8vPUgAsoSmXOJCJO8Ekr6AvV8aiPibZwuOnYI1vxopiusNCpB0T66iVrWvv7zyDJHRlI++3iEWchqLL7rvF7NPxYLHvDVlKlhpEHPv70mPbLnX8TIL8manQ1YsGFPx1YmB+QPPpuB0msffBvffmQ5NtnrowaRamfNP1/Zx2vEdY7lH1/doFzPPs434Os+xpJ9PNE+EeL8M7JPptyVh19Tv1wMP+DP569iMv5B8fh6zz/cqsYNoZ7+3D7Bcxv0s48zxwd951n2YbLix0+t6ugqZE0uss/2A02uY4Nob9kn38t5DYLOuCzclFkzKe00np7zpIWrwYsTuTx/neZ/lCdUM+HybDP7JhTvuZBGysnnrzX+GTOltj3aBGzyfBCu99axEYw/kcFwu0M0wzx/03iErH9nkH3Y+BcxLjmjCBPkOLJPFtZfDnhG+SG3v+af3/W8x4fN9Mw0nn7ZRy2KxbwDvv75AoN7ucM04/GMB6zuWu6SfQKrlxXSWOukHJfmHzPLTmpZxvPXRwqZCJ+5K+sRnMk0VPMP+qqw7MOEoRqL9soHUkik8Vq78xA21B+KdIy857DcNIB+QlRbkq/m7zX+QohADzEz4Ove53BLCrWb9znv1cXB5f7e5+ddVtHS/P3nV/craLCPjPYJ0/zVAd+AgdnMgvCZbVHSJnjTVnghx/jHtNE+FYkYrj51pFHzzxh//+ca1PtpC9j4FzGuwcEOrEeuyB/oGxv24NxfvBLpGKlVRzHs7R/qmd/xXtknmRaB2rAzZuK5zxseXe56H48R+nevcN7/7Nk1ONLqzqLuzcqp5qsXnnKJdxA0H6Jq/jGX7OM/j7fO2Q72e/P0mzBF+/zvuUfj5lmT9esAI9NbCHIUWPbpwsz67Wu4/tFleZ9HNQwduR5uU0sKo6570jUYGYVsbPO6XYfwlYfeRlOLFaMdJby1/TX/3K+XSvu9/FQ6OCrHFOqpW6B8/ODurvcHmlpd78sT7u+PGu8eFo1VEE9VI9M4dXFN8lJ7Hubr6gZ8o+BN7PbY3NNdE9vUHop+0RerTOv5xzIrfOk+V2dSMnv+XZeldQfwt4Vbw3cMIaV4vx3p+e84eAQA8AeD8f/fv72Nc37+sq88myifr89bgseWbHNy0usM3IINe3DjPzNeb0dq/m9v2efb3ticxFk/fQmvrd3tKn9vfxOOuuEp/KO2zlWeTInAqJxFm/dh7c6GSFFGYwf1cL3faX9mEu8avC7N33B+x/MvgOsvNf+gCB5re/B5HM1fM+AbBe8avlOG98a1545V6inroff8g2bwqmk2dI80I/tkWekcYeNfxKj2r710Qh0yAkf+NBZt3ou/LtjsbP/X0m3YuLsR+w+3uIyO12h5pQiVJs82nec/+64FeODNLU4KhLBwxkKj3s4H7ngdL63ehT2Hmp2yV9+tx5a9h3HV3W9i7oOLnfK1OxsAAI8sdhv/VDod2IB99v5aXPCrVyM1cj0ry3DLBzJpmX3G3/P1UbNWms6eCX8MvXwojkcd8j0O+557575k2zCFJWxzchBR2GQw/fHXv38CPn7KSFxy/BDjNTjOn3FoTaWx/3CLr9yle3eQ7T/SmsJH//gmgMwP7oN3voHvPLbCl3v+4ttfw8m3voB5tVavxyvb71YMpRfvAKX33L97aV2mTnbWxnxkGCEEFm3OhFXOfXAxfjn/3cBjvNe75t63cNXdmeWplyprCT+5bLvzWnqJXiOeDJF9MtcN3QUA8LGTRzqvdx10P+s9je7vV9w14Ks/n3fhlHyQlwjLaBm2YljMU+dcZJ9g42/9j5oADgDG9K92tvWpLscPL5+snQ+ROV0Xl32IaCYRrSGidUR0XUfVoxj4+rylmPKD+b7yzhDts01ZANxrIzZ6JmLJxcK/9bA13uGVfXYfasFNj6/A7S+s9V2nxZOGV5V9Dhxuxc+eXeO8lz0I2TvYurcJCzfuRTbMq92KD975Bp5ebhnpJ5dtx+0vrMXdnrVaJR+/+03c98ZmX/nqHQd99ZIkU2nUNzTj6j9bDYT3eaTS4dk3gewmlz3yhVMBAA2e/DbjPLJQIhbTzvBVkV+5ghj/gKyeKmGNQybUU8o+2dUtLNpHPolEXJ/bx0n9bF/3axeMQ69uZa5txnOXguxDRHEAvwPwfgATAVxJRBM7oi7FgCkXiTvaJ79vTNCgXsORVjy3cge27vVnD922PyMfSAM9rHcVAGDVjgaf0Zbsa2zBw4vcMkdzawr3v7FZ62EHyT4bdrvDRKWRVZ/PtYrMIvf52bOrUd+Q8YD/s7YeL6yylk5cX281XFv2HkZzMnPtH/77HQDAb19ci2dX7lCO3Y1V2zOGXqJGqnjz6TQ2p7DivUxvQNb3suOt5GlRPf+gsZOHP3+q632vqjLfPitvnoEPnljjKnMZQcPpvatb5YPj+SsGctaUob79dI3D+RMy6xz7on2yrJuq5evWKJDfrfJ4TCv7ZDR/670Qme9A1Lp09QHf6QDWCSE2CCFaADwEYFYH1aVoCFquL98FIIL08WO//xzm/GURZvz6Vd+29/ZbDcLwvlU41JxEKi3Q2/Z06huasacxY1yPHdYLv5k9BQDwk2dW+86lq8JtT6/GnPtrfbKX6hFvqHf3MBzPXzmhlJTu+e9GzH1gMR57+z387qX1uFWZGf3xuxfi0/fVus4lAOw84JZI1uxowM+fexef+0t4NtXWlHDGRLyGvKG5FYdbMg2LbNDOHj8AI/p2szx/pZEb2kufL9/UOejdrQzTRvV1lfWs9Bv/6oqEzzC5JyQZPH+p+RdiwFcT4/6b2Sdg020Xu/bTTdT609XT8M0Z4111ynj+bj73vjGB9YjZoZ73XnMS/j7nVN922Zsd3b/ayTY695yjfOsNqL0mKVWFPSYn2id4t4LRUcZ/GAA13KXOLnNBRHOIqJaIauvr6wt28U/d+xbu+a+++96Z8f4GC+r5G37gqterGqp0WuCp5duxcttBEAFX2Xryoeak8wNtONKK/YczIYXxGGF4324AMhKQik6++P0r6/HcOzs14YyZgq373D2SI61pbN172CWzyONv/tc7eHL5dry61vo+qZ63CW9d1UZw697D2vEYlTN+8hKeXr7dN25xqDmJg0cyz0c2aPEYIREjn+dvSmZn+uxSmkHxnhrPHwhOlRAW7VMeFoKTBWGyT7dyfS7K940bACCTWdQU6nn9+ydgYI8KmJADzmePH4gBmv22H7B6uuMG9XBSZnQrT2CAPY8is44A7HoIJyVGmEff3pO8Oiqrp+7ufN8wIcRdAO4CgGnTphUsdGNZ3X7nwyoE33p4KdbsPITH555esHPqSAuBmPLo3Bkg8zu3SV7YfUhv2O55fZMjgfTpVoY+1eUAgINNrY5hPtiUdOVOj8cI3Susr5xODoriSUvkgO+dL6/Hr593jxHsbWzBJf/3WuDxUq7yxrsDwEurd+GVNRlnY1+AcT/zpy8FGhPAerZfeGAxLj7OHeHRcCSJ3YrspKauiMcIyVTa1XsxGWFTua43V5GIIUZWY3jl9BH43qWW2jpuUHecP2Egnl+1y9nXmyTNizRwOikpV8KifXpU6k3W5GG9XL2EoEleUdI3mPjItOH4e+1W9Kkud2Y7E8FZJjPu0fyFcr2oP9GuHu1TB2C48r4GgDnJdoGxBtIKFwY4r7YOS7fuL9j5THhrXMh7MJ1L1cRV/vl2Rq/vVVWGnvaP8ncvrXO074NHWnHIY/wr7SgH3fW8g5CB9bUN5Z0vr/NtU6UmFbVnIcMrG474r3nNvW9hjb0dcPd+dOwyPCMvOw64wytfXrML/6dEKe21I276diu3jH9auAy79Gq9vLtTnxpDP5GIUFVmfQbdK+KotF8TEa6/aIJr36MHWhPD3jd+gPb80kiZehO5QCEWqdrg+XsJmuQVaPxDrP9PrjgO62+9CIC7xyN7JE4acY3mH0Z7R2t3lPF/C8BYIhpNROUAZgN4or0u7v1RFQveKqvGJN+uomnAd7fGsLWm0ljxXmZws2dVGXrYWvJDb2XUvPqGZnzynrec93EiVJRZXzlvqKaXG/+5PDDuX0b7JDSSw95GvzcPWD0RiZws1tSaQmsqbYyYEcKSkQrBNo989LuX1mt7QP17VCARJ5fm//urpma9SIopUqjCNviyEZB4ve6jB3bH0psuxEen6+Um+RH2NHjjuRAa5x/RkMrft05qCTLGYdcHMo2H9PyTKYHqcutZNtnfq8xYuXD2D7M57W2SOsT4CyGSAK4F8CyAVQDmCSFWttf10xGjKNqC+e/sdF7LyJKoqGl6tx9owi3KYGXeso/hm1evib33esuW5+/3/l551z1Ok4hnPP9mQxSQ5IE3t+D5gOcjGw/dD3lfo16m2d3Y7NpfyhUNR5JOY6CjOaARyobtHs/fxIDuFYjHYnh9/W7nOxLmPXoNOWCOz5dnqiz3GH/NNXp1KzNq1Q32eEUhPf8o2VqzQXe2wLV3s7i+nK3bmkqjm/0sG+30I5kcPpm5CVGSEbYnHRbnL4R4SggxTghxlBDilva8dkoEJ8xqSz57fyaaxBtZEoZa5dpN7vQB+Xr+psbwkEYWkT/6Y21PtDWVNmqxKjHF8z8UQeIJaqCTaYEN9YccT0tlr0GjP+8Xr7jkplH25JtX363Hk8v0qqOAcCaNRWFwT31ETlQSMUKvqjKUxQhHWtP4hR32KtManzSqj/a4S44zzxj1Ir/7cvxFEtXwvfLNs/HC19+Hg/Z4SSE1/0INduqyekqCGtJsejFyILcllUaVLfvIoAhV9omyepeprm1JSc7wTaezWzqws6Aa/zU7Glzb8v3emL6YXumlJZl25JNjayzjv/9wayTjn4gRKhIxEMExHEF4exjqj7Yllca5v3hFO06w1zBI7WXiEGti01f+vgTffmS5dp9NuxuxeY9/foOOowZU570EZu9u5fqc8vbH86dPnORIDJLyRCwrZ0Z+1l79vMy+5nRPeKiXkf2qcdSA7jhofz66Xl+uFMrzzwz4+glqYGTgQhTKVdnHHvCVAQ6ZtYSF870NM/4lIft0NClR2AHffHlo4Ra8tSl8Bqoq+3ijVPKdGGI0/p7Bzv1NLXhnuxUeOXmoZfz3NragX/cKnH50P2c/XfifjKGuSMQiDe4+6sl1o0obP30mM6P30uOHujxik+fvZebkcG95Xm0d/rZQv6i9t8H7ziUTfXnxvUwY0hMXHzsEl3smMJ082jK4coF0dcWsL517NM4Y2x+AJcN88vRRrmMrE7GsnBm5b7XH8x/YsxK/v2oq7rnmpEjn+dC0GhwzuAc+dkrh1nwulOqjW8xFkghYy7dfFsZfftYtyUzPV/6OhthzMob0qnR6VB0lNZsoTeOvSZ/bkVz36HJ86PdvhO6negZeT6+tQj29g52fua/W8ZInDu0JAPj0GaMBABdOHOzs5w1rvHzKUHz/skkAgMqyeCQvZ/GW/a73ph/tiL5VLgnDG1Xj5ZzxA/DKN8/2/dAvPd4/ozQIr8dbkYihsiz4J9WnWxl+97ETMX10pqGcOrIPbvvgcQCAL559FIBMls3LpwzF1y4c72pUPnPGGMyYlIn8qSiLZ+XMmGQfAJg5ebCvUTAxpFcVnvnKWRjSqyrytcOI4sRcc/oonDVOH4EkkT0I3XfGm8FUpU+37I1/ayqNi48dik+eNsqZbDZrylD88RPT8IlTRzmef2dyOIESNP7S6EftJtdu2ovfvujPNaPjQFMrPvHnhS7js2jzXoy67klXjpdcUWvsbbzyln0Mz8ObVmGZkpysf/dybLrtYnzufZbBqkhkvk4/vHyy67hfzz7BSftwuDm7AdQfzJqE38yegtH9/dPtAWtArVy59hZNGgqVwb2qMLJftRPKKDHNoNVx3jEDMayP2+hVJGKuBuHiY/09Cxmd1M0j3YzuX41F3zkfHz91lLWfbTCqNKGNfarL8YePT8OvPnK8c91svEq5b7eK3BZbbwtuvmxSZK/7e5dOwv2fmh64z9cuHIdrTh+FK6bW+LZ9zBC9BAD9umdj/DOaf3kihu9fNgn97PlDRIQLJg5CLEZOA5Rq55Xlwig54y+NXNQfyxW/fwM/fy44m6Pk8SXv4dV3610ZJp9abuV/+c+7u02HRUYNFfPWP+/FwyNq/io9PJ6vNMBHD+zu8iqf/vKZrv10ufiD6FVVhllThhnvsSxOoXKLimykKj0RMlH13h/OmoS7P3kS/u/KEzznjTvd/xF9u+F3HzsRN1x0jGufjFH3G95+ysRDOfDobSRUBvawGqvjh/fOzvNPmz3/juLq00Zh0XcvKNj5elaW4XuXTtJmz/zwScMxcUhP/XFZDF5non2Cn72M9mHPv4ORX/xs4/y9SbmCEJpMWAVZ7EiVfXzGP/O+vqEZW/Ycxuf+UouLfvOfSOc2Gf/m1jRq+lThOxdP8G3r4TEeuh8aYOncJnTJs7zHyPOaYvET8Zh2jOGnVxyHi44djKU3XeipZ2bfhTeehzNtPT3qRyRlg0E9K3G2MgGqIhFzjIfsMXmfiWP8NaGZKr1t+cE7uKty2lH98Mo3z8avPjwlO83f3jWqvFMK3PXxqbj+/cdkNfNffudaQ6LBokb7tDcl9+lLIxllGUCVwy0p9KoKbit1P8CCjuAHaP6qV3HSLc9nfeogz79XVRlOGNHbt80bGqhKL1EZP6iHLzEbAJw5tj9S6TTe3XnIMdamR5mIkRNi16MygYYjScQI+NDUGnx42nDf/movYWCPSnx42nD8Z+1uX0/GhBp1NHFIT7xsp4KoSMQd2eeIY/w9C6PHZT6YYOMveyVBz5SIMLKf1Xjm4lV2jzhbthQY1b8aF04aHL6jQkLR/IOQg7+FDIktBF3+029sTuKfb7+Hj508AkSUtedfHo+hJZVGU0sq9MNT2xMhBJ5ftStwpqGsX1TUHoXP88/Tq9AZ//f2N2H3oWZUlsV9EokOx0iLzKzUd3c2BB2Cvgappaos7hi+oKXvAMsYywifYb2rsHpHQ+BC3N50DZccNwR9upXj1KP6YfKwnkilBT5wx+u+e5MT09RG76sXjMMdL6+3y4GeVdZPyjH+ngFgKQGEyVTyWbZEdFJymbFe3Yk0/44ml465bMDDnJ5rTh+NAT0qcOlx2QUUtDVdXvb50ZOr8J3HVuA/9rqpcswlahdMRm+c8uMXQveVAzr7D7fizlfW47P31zqrVpmY9L1nI9UDcOv63h97lEU/gvD2JIQQOP22F7G07gAqy2KhMgXg/xHMnDwYXzpvrGFvi4s0g6KAZZhkt1oayq9dOA7dKxI+rzkRjzmZNYfag8pBWU698weICGeM7Y94jHBcTW+cMMI/kWrqyD5Onn3VbpfFY7jd1v77VVc4vQepA5fH9bJPmAwon2VYXiHJx7KYX3DXx6fi3GMGalNjlCq5hEqfOqYfvnHhOPzIE9zgJR4jzJoyrCCprwtJl//0ZWKyJifHu2UkU8JaAPuT9yzU5pm5/41N2Lyn0ZhCVof8wf972XYnDv2wJ9dHPoigAd808Namva6Ea9ngPZ+6rF9lIh7pOXglDhPzv3qW8/r0o/vj1W+e47yX8frdyhNK7hTr8zln/ECsuHmGry5lccI+O3X00N7hETu6ZG5evnzeWJx3zECce8xAAFYDI5+RN1TwsuOHYtNtF6OqPK4ZB9HLPv1DtGXH8484u3jm5CGurJbDelfh2nOO1u574aTB+PMno8Xyd1W8tj4XuxyLEa49d6xroL6Y6PKyj3dRB+nhptMCX37obdTta8L2A0ecPPOA1WW/6fGVGNSzwmVompMp46AmENybKMiAr+tamddnHN0fh1uSkeYKmPDWff2uTKbIyrJ4Tp6/ibGeJQMryzPHffWCcXho4VZ84IRhzkpZ3ugg77NMxGLo060M9Q3NkWLO1Rz6Jr56wTgAwB9eWY8XV+9Ct7K4EwYYlAIgFiN89szROPcYKw7fJ/vYDdrQ3lW4+bJJ+N4TK7UD2acdZQ1Cy8YnKrd+4FgcPbA7po8OnqXLuGmv1bM6E13e+Et5xFmVyLYjybQwasiyfOfBZgCZxGaNzcHGP2jQrTCef+a1GjMcjxHyzRnlNf7r6jPGv6Ispg1N9JLroh7qdPvBPSsdGeXyKcPwn7W7cdSA7qZDAVje9L3XTEft5n04IGf3Gh749FF9cf37/ZFLJuRYR8+qBK5//wRMGNLTWTjExI0XZ1YkNUX7AAi8r4lDe2LDrRdlLRXkm16iVOlkiky70OWNvzRpsqeuev7mY8xJzkwDlEDwJI7lSgrkXHEN+CpVjMcorwkkr7xbj+V1+11l6zyef1nAlHh/PcP5/VUnOgOoahpdNbfLB6fW4NLjh/p6FN6alMVjGNq7Cpf1rsKDb27R7iOZ93n/0nxBSLmwV1UZqsrjuDJggpAOb93VBjzsmXY2jbgr016rZ3Umur7xlzk+ID3/8Bm+pnZBZqJMpQVi5O8qBoWPPrK4DpccHz3z4ourd+KX89/F43PPcMrUKquNV4wIWc6bcnH1nxf6ylTj38dO63vVKSNw/oRB+OQ9b2kX184GNa+O+sPz/gijSElqg2HK2HjvNSdFWrLRi8yhlGvyMq/m713chukclKDt7/oDvtJESlnDCfUM8PxNYXONLUnsOHAER93wlDaKJyzWetNufzy7yu5DzfjLgs3YfagZX/37Uqx476Ar+6VuktfXLhiHRI6efyot8Oq7/rWR31i/x4mOAjKDkz+6/FicPX4gln7vQvziQ8f7jhvVvxrnHjMQv/rwlKzqoa7eFJR0y4TqQZuOP3v8QFx7bnDkkQ4Z3juiX7eQPfV4jX+DMt5Qigans9IZNP9P2TmyhvfJ7buWLSXg+VtGUk7EkB6/aqi9n7sw2NFDR5JYuc3yHp9ZscO3oLZuwFeulwpYi4cH8ejiOtz61Gq8sy0jEal1c8s+AieP7osvnTcWcx9YnNPswbte3YCfPLPaV37lHxe43nsjU0zzHcrisZyiSFyyT4QfoW7A1zm+wN70p88YjaG9q3BpFvnyVSo8A+VRIo2Y9qczdMKumFqjzUXUVnR9z9+2ia0ejz/IWJokoUPNSSekUJf9Txdrn40xkoahsTmpjQDxev7Sy7U0/4AxDCHw7Mod+OVza1y5ejYog7pBZJPsKhdcsk8Ov0LV21cX1S4EZfEYLjt+aM6eoXcQnI1/56QUNf+8jD8RfYiIVhJRmoimebZdT0TriGgNEc1QyqcS0XJ72+3Uxv0t6S3L/BvOgK8QxlwxXtlHygqNzUlnMpHMvfL7V9Y7+7UmdTl9ot+enBOQFhkf36Xze+L85RfWivYxG/9/LduOz/1lEW5/cR3++OoGpW7R6hV10excUcPmoy52raLOlu1sOro31NO9glnm82Pal/891z0HohQ/gXw9/xUA/gfAq2ohEU2EtSj7JAAzAdxBRLL/eyeAOQDG2n8z86xDII7nL2UfjefvNdBe4y9j3JtaU9hrT37q082SPm57OiOb6LJVZvO7lrM5XfH8Sl28DYE0GvEYIRUw2PzevszC4XJxln8v24aX1vj1fpUpw3sDMCdfKxTZev7e2btqg5FNVFJ7ID1/md/l46eMdLZNGd4bnzh1JH6Z5RgJkz9yUpycs9EZNP/2Ji+XTgixCtA+uFkAHhJCNAPYSETrAEwnok0Aegoh3rCPux/A5QCezqceQaQ9mr9UZqxy/QfudaLlxJy0gCP7dK9M+NId66biZzMO+9cFW+zrC6cF0A0+r95xEMvqDuA8ewJQuZJ3RoduBvO1D74dWp//OXEYHpt7epSq50UsS83fi9vzt2WfTuLLVZbF8auPHI+TR/dzUk9I4jHCD2YFpwZg2paKsjhwJJlTbqRip600/2EA1HCYOrtsmP3aW95mpB3PXw70pu3/mQ978x53FI5XP5cedjotcKApk/Zgxq9dHR7fqldA9EVjVBTb72o85Klm/tpK0yy95B4VicAF0bNJR60SJZlbIVCd/VwkkCihnh3JB06o8Rl+pnMgU2ZHTaPRlQg1/kT0PBGt0PzNCjpMU2ZytY3WkYjmEFEtEdXW1wdLFEY8so9s4VPpjK7+0T++iQOHMyF4Xi9AGpS0EI6B37znsG9h7xdX7/JdPpcoHPUYl+zjeVTSS+5RmQj0/JsV41+3r8k41uElSkqHQkCGSV7m/d3vXaGendD4M52XP3/yJHz2zNGo6VN6jXOo8RdCnC+EmKz5ezzgsDoAaiL1GgDb7PIaTbnp2ncJIaYJIaYNGBA8pd54DrhlH2kH0570Dmq+F69tdDx/kfEQ7n19U071iYKVesI/H+Hi21/T1itsRSZ1IPrxJdvw2JL3ItWjvYy/SqRQT897NdQzl3kCTOkyZkB33HjxxJLU/NtK9nkCwGwiqiCi0bAGdhcKIbYDaCCiU+won08ACGpE8kbaTpkX3RnwDVgMJcjzb4/uYWsqnZF9lLp4pR25LWwRkgfe3Ox6/9raPZHqESWfT6HJN9RTNgQl+FtmmKzIN9TzA0RUB+BUAE8S0bMAIIRYCWAegHcAPANgrhBCjoZ+AcCfAKwDsB5tONgLZDz1VdsPYufBI47B9A7EqjNkvUqNqvk355NHISLJVKZXEqQayQare0CWSQA+SeiRxdHSPkdN0dzRdOZQT4bprOQb7fNPAP80bLsFwC2a8loA7RbiII3//Hd2Yv47O3H/p6YDsAZ+VQ29NSVQt+8w+nev8On00ptMtZPnr04W89ZF1eulJNQjxPjnSmftCnvrpUYLZbOQO8OUMl3+l3KxZ1p+ZpKXe79UWuCMn7yEz9xX6xsQjbk0/2grK4XRt7rcGD8fJEGp9Zb3kmvSsa4Ie/4ME40ub/y/dN5Y9FfSE6RdhjWznxwQfm3dbl/DECM7R09aaCdy5cKPLp9sHFBNpoTTK/Ea/5Sm/mEDvkF8ZNpw18QjlZE5JjNrL+SCJer9c7QPw0Sjyxt/wC0FuMIoldeqLu41uESWR5kWQpvCIRdiZDZUram0o/n7lmvUyD4DemQSr82cNDirevSpLsfnzz7KV770pgtDlxrsaL49czxW/3Cma2BaDv5yE8AwwZSc8VeNpzrzNcj4x4hAZOXPKZTnHyMyShTJtD63j/e9HBuoVjzfY2t6ZVWPsjihTFOPzhwyKSX+RCzmm4iWifbpvPVnmM5ASRh/1ZCptlv1qtVUDd5IIIIVfy4EfCkd8qpXTP/41UYp2PP3H3tclsY/EYtpG6Fi0M51dezMjRbDdCZKwvir0SCuSBphMP4+2YcQI8sQm4x/trZSALjUsBpWMp3J7+Cdj+DK7KmZqXvm2AG495qT8PI3zsY3Z4wPrUciTk7uIpViiJrRpeFlzZ9hotHlF3MB3LNGv/zQEue16jkHG38r4qc5mTLG3SfisazDQK86eQTu/e9GrK935xZSB3y9kUfq9dV63nvNSc7Si2ePtxK+HTO4R2gdyuKkNZid2fN3ZB+Nl8/r3jJMNErC+JvkX7UX0NSiGn/3fjEixIicfPs6yrM0/kJYPYpumlz57jh/9zZXtJLy+uzxAx2jnw2JWKxopRKd5y8b+qG9K9u7OgxTVJSE8Td5saqRf+itTBJSX5y/He0TtAZvIfPIt7pm+GYn+6jI+QJTR/bBos37tPucc8xA49hDZ4UCFkGprkjg9itPwCl2GCjDMHpKwvhHWaJt9Y4G57V3kJVgaf6Lt+xHt/K4tgeQvUZuNtxJNbePb8BX3S/E+NvbB/bQh2wuvelC9OpWvBPETArPZcfrx1IYhslQXC5fjmSrA3tlH6JMAzK4l15OyHWAVNcuubJ6+kI9zbN//eexNCNTzydepHKPpATX32CYglEaxj/bSBzdgK/MnW+YTZut7BNkuNT0Dv5oH/1rHbIHY2qYijUyRjaYbPsZJndKwvhnuzSgbsBXes+m9Mm6cMkgggyXutCMX/bRD/jqkLKP0fM3lPP8KIbp+pSE8Y+i+at4vW2ijEE05dFpDFhGMQhTzQox4HvyGGvQ80NTa7TbTY1its+rvelVZTXAuaz3yzCMRWkM+GbZxOnSO0iDaMqdn09ytSD+tdS90Fk6beXZb06mcU5IaOfIftXYdNvFxu2msZDOrgb94eNT8eSy7Rjet/SW3mOYQlESnj9lmeZLt8Zt2JKJJ43ui0e+cBr+58Ro69FHHax8bInH+AuBHpUJnD9hIL5z8YRoJ8mSXNYdbk+G9KrCZ84cw/l7GCYPSsL4Z4s3Z06MMs2HaeEUIQSmjuyT/SBq1uMRAq0pgWG9q7IaZ3jwMydncY2sqsQwTBHCxl+DLr2DTP9gNv7WfykPnTm2f+A1RI6xKmlheebZDjCfdnR/zDLkEmIYpvTIdw3fnxHRaiJaRkT/JKLeyrbriWgdEa0hohlK+VQiWm5vu53aoe+e7RV0mr9M+VylSccAZIy/vNYpY/oFXiPXGHUhBJLpdE5hmr+ZfYJx2xVTa3ApT45imJIhX89/PoDJQojjALwL4HoAIKKJAGYDmARgJoA7iEgmXr8TwBwAY+2/mXnWoeB4FzwnZCJrdLnvgUyDIduytkqMJj3/Qp//5x86Ht+4cFxBz5kNz331LLzw9fd12PUZptTIy/gLIZ4TQsgYxwUAZEzhLAAPCSGahRAbAawDMJ2IhgDoKYR4Q1ijqvcDuDyfOrQFrSmv7ENOTL1JbpFHSJscZpvPm2BF6mRrwlNpS/Nviwla1W0UsRSFcYN64KgB3Tvs+gxTahRS8/8UgKft18MAbFW21dllw+zX3nItRDSHiGqJqLa+vr6AVQ3Gm52TKDMIaprJ69X8g2Llf3rFcdpsnlGQKRuy1fyjUJ1jnRiGKT5Cf+1E9DwA3cKwNwohHrf3uRFAEsAD8jDN/iKgXIsQ4i4AdwHAtGnTco5ByVbzb/XkUY5RRtYxZcCUA7jS6AcNZeQziUrWrS1kpcoyHv9nmFIh1PgLIc4P2k5EVwO4BMB5IhMgXwdguLJbDYBtdnmNprxT4fP8QU7suzH3vTcZXMD51W2yHZgyvDeWbN0foW6yESq88ee4eYYpHfKN9pkJ4NsALhNCHFY2PQFgNhFVENFoWAO7C4UQ2wE0ENEpdpTPJwA8nk8d2gLvIu2xWEbWMck+smcgvXq1LfjFh47HD2ZNcp3Py3cvmZhV3TrzSlsMw3R+8hV5fwugAsB822tcIIT4vBBiJRHNA/AOLDlorhBCJsH/AoB7AVTBGiN42nfWDsYf7UNOtI9Z9rGQNlmdJfzBqTXYc6gZNz2+0t4nK/XLRatdt7ZaY/e69x+DMf2r2+TcDMN0HvIy/kKIowO23QLgFk15LYDJ+Vw3W7JN7+DV/EnV/I2ev/Vf5svRLQKvr5ubYb2r8N7+ptC65er5P/z5U3HF798wbv/8+47K6bwMwxQXHN6hwR/tQ4rsE+xxSxufFlbsujTRqq02Dfi+/d0L0JpKY/qtL5jrZhv/XDX/aaN4eUOGYdj4a/Eaf9XOmoxuud0oyF5GWgiMG9TD2a72Pkzjqn2qy500EiakJMWaP8Mw+cCxfRqk8R9vG2/VzJo8/+9eYmXYzGj+7u2kHKbz/OX+5SE9Cyn7tJXmzzBMacAWRIM0sGMGWAOfqrHWaf4XThyE3t3KXft600Kr51Cd9vKE3WOQM4NDPPpW9vwZhikAbPw1NKc8s2hdso//kalmPqZo/ipkePerj0zBZ88cjROG93HtccHEQdq6ydQTxbr+LsMwnQPW/DVI2UcaWNVrD1uoXUb16DKDZl5nyof0qsKNF7tj/Nf8aCYIhHHf8UfBFiLOf/5Xz+KeA8OUOCVh/LOduNrixNLbqRqUbWE5dcjk+ZP6OrhCFYm4djUxIDPgm4/mP1YZiGYYpjRh2UdDq0f2cXn+IR6zs69mQZjMPuF1MDUQbZnbh2GY0oGNvwaf7KM8pTDP36z5q7JPHondkvnF+TMMwwBs/LVI7zoTdhkc7aNi1vzVnXKv259e2wiAPX+GYfKDjb8Gqat7wzABoMyQ20cSc4y/vtz7OoifXnGccVtb5PNnGKZ0YAuiQUbUVCSslSfVwddwzx++Y9RyILrj3797uXEbyz4Mw+QDG38NTrRPwjKwSWVZR9XoHj+8NwD32K4zw9dzTsrB8w+KCmLZh2GYfGDjr6E1lQZRRuJJKdZdNchfPNufAVPO9O1VVWY8f1S7HdRIhPVAGIZhgiiROH+/oYyRX5eXNLWkkIiRY2DTph01XHFiDdJpgQ9OrTHvFNn4m7eZ1hVgGIaJQslakCDZ5FBzElVlcWdQNWWw/UN6VQIAJg3t6ZTFYoTZ00cETsKKKvsEev4s+zAMkwcl4fnrsAyr3qqnBVBVHncmdKXSaUwa2hMrtx107XdcTW88ce3pmDS0V1bXjmq2g9oI1vwZhsmHkjX+YcbT5fmnBf7x+VPRcCTp2++4mt5ZXzssc6ezH2v+DMO0Efku4P5DIlpGREuI6DkiGqpsu56I1hHRGiKaoZRPJaLl9rbbKSzRTQHQXSAectnKsriT2yeVFuhWnsCgnpVtVh8dwbJPySp2DMMUgHwtyM+EEMcJIaYA+DeAmwCAiCYCmA1gEoCZAO4gorh9zJ0A5gAYa//NzLMOOREP8ZyryuOOgU1lMeAbhajtXfCAL3v+DMPkTl7GXwihiuDVyIjoswA8JIRoFkJsBLAOwHQiGgKgpxDiDWHNgrofwOX51CFXTJ6/lIMs2ceO8y+w8Y9qtwPj/Fn2YRgmD/LW/InoFgCfAHAAwDl28TAAC5Td6uyyVvu1t9x07jmwegkYMWJEvlV1YdLde1YmsO9wK6oU2cebpydf2PNnGKajCfX8ieh5Ilqh+ZsFAEKIG4UQwwE8AOBaeZjmVCKgXIsQ4i4hxDQhxLQBAwaE343xHvxlJs9fTs5SZZ+kKdYzRwoxyYujfRiGyYdQz18IcX7Ecz0I4EkA34Pl0Q9XttUA2GaX12jK25Xq8rjRePbqVg7sOeySfQru+Ucc8g0y/mEJ5hiGYYLIN9pnrPL2MgCr7ddPAJhNRBVENBrWwO5CIcR2AA1EdIod5fMJAI/nU4dsufGiCXj5m+eYjb/i+cuJWoXW/KPGNwXtFzVclGEYRke+mv9tRDQeQBrAZgCfBwAhxEoimgfgHQBJAHOFECn7mC8AuBdAFYCn7b92Y+yg7hjQoyLc+JfFHV09m/QOUYhq/PNZ9IVhGCaIvIy/EOKDAdtuAXCLprwWwOR8rpstrnVUSC7Krt9XBtG4NH+P8Z9z1hi8+m59zvUpj5iLn5UdhmHaipKY4VtZFndeS5tv8vxl43D+hEGO5+2N87/hogm44aIJOddHLhIThsnz/9tnT8n52gzDMECJJHb70eWZjoa0pybDesNFE3DvNSdh8rBeTgNR6ElecpGYMHR1/Nz7xuDUo/oVtD4Mw5QeJWH8+3WvwGm2wZSRNqbcOAN6VODs8QMBZHoHqQJH+0T3/P1l1808pqB1YRimNCkJ4w8Ap46xjP+Q3lZ+nrDcPgDa0PPPXfZph1RIDMOUACWh+QPA3HOOxuUnDMPwvt0ARAuVjBs0/3zJx/gzDMMUgpLx/GMxcgw/EM3zl9E2hQ71TESM9mHbzzBMW1Eyxt9LFM8/oVnDtz3hiVwMw7QVJSP7eImSGG1gjwrMPmk4rjplZDvUyA/bfoZh2oqSNf5REqPFYoTbPnhcO9TGcH3WfRiGaSNKV/YpAsPqrWIRVJlhmCKhZI1/MaRELoYGimGY4qRkjX8xmFWv8S+GOjMMUxyUrPHvmPid7CiCzgnDMEVK6Rr/DgrfzAbvbF6e3cswTKEo2Wgf77ytHhUJfOGcozqmMgbY82cYpq0oYePvtv7Lb57RLtfNZuF1r+Y/fVTfQleHYZgSpWSNf0eoPou+c37k1A6A2/g//eUzMUJJT8EwDJMPBdH8iegbRCSIqL9Sdj0RrSOiNUQ0QymfSkTL7W23UwcJ2YVelD0K/bpXOMtERkF9MhOG9ER1Rcm21QzDFJi8jT8RDQdwAYAtStlEALMBTAIwE8AdRCRXMLkTwBxYi7qPtbe3O0Uw3stx/gzDtBmF8Px/BeBbcEdPzgLwkBCiWQixEcA6ANOJaAiAnkKIN4QVbnM/gMsLUIes6QjPP1t4wJdhmLYiL+NPRJcBeE8IsdSzaRiArcr7OrtsmP3aW246/xwiqiWi2vr63BdM11EEtp89f4Zh2oxQEZmIngcwWLPpRgA3ALhQd5imTASUaxFC3AXgLgCYNm1aQc21KIJpXmz7GYZpK0KNvxDifF05ER0LYDSApfaYbQ2AxUQ0HZZHP1zZvQbANru8RlPe7hSD58+TuhiGaStyln2EEMuFEAOFEKOEEKNgGfYThRA7ADwBYDYRVRDRaFgDuwuFENsBNBDRKXaUzycAPJ7/bWRPMWj+kmkj+3R0FRiG6WK0SeygEGIlEc0D8A6AJIC5QoiUvfkLAO4FUAXgafuv3Snwyoxtxgtffx8G9azs6GowDNPFKJjxt71/9f0tAG7R7FcLYHKhrpsrxZDbBwCOGtC9o6vAMEwXpGQTuxWL588wDNMWlKzxL4ZoH4ZhmLaiZI1/Op15/fULxnVcRRiGYTqAkk0WI6N9nvrSmZg4tGcH14ZhGKZ9KVnPX473xkr2CTAMU8qUrOmTnj/xyrgMw5QgJWv8q8qtJKNZpNdnGIbpMpSs5n/Hx07Ew4vqOI6eYZiSpGSNf02fbvjK+RzlwzBMacKiB8MwTAnCxp9hGKYEYePPMAxTgrDxZxiGKUHY+DMMw5QgbPwZhmFKEDb+DMMwJQgbf4ZhmBKEimVFKyKqB7A5x8P7A9hdwOp0BF3hHoCucR98D52HrnAfbX0PI4UQA7yFRWP884GIaoUQ0zq6HvnQFe4B6Br3wffQeegK99FR98CyD8MwTAnCxp9hGKYEKRXjf1dHV6AAdIV7ALrGffA9dB66wn10yD2UhObPMAzDuCkVz59hGIZRYOPPMAxTgnRp409EM4loDRGtI6LrOro+QRDRn4loFxGtUMr6EtF8Ilpr/++jbLvevq81RDSjY2rthoiGE9FLRLSKiFYS0Zft8qK5DyKqJKKFRLTUvoeb7fKiuQcJEcWJ6G0i+rf9vhjvYRMRLSeiJURUa5cV1X0QUW8iepiIVtu/jVM7xT0IIbrkH4A4gPUAxgAoB7AUwMSOrldAfc8CcCKAFUrZTwFcZ7++DsBP7NcT7fupADDavs94J7iHIQBOtF/3APCuXdeiuQ8ABKC7/boMwJsATimme1Du5WsAHgTw72L8Ptl12wSgv6esqO4DwH0APmO/LgfQuzPcQ1f2/KcDWCeE2CCEaAHwEIBZHVwnI0KIVwHs9RTPgvXFgf3/cqX8ISFEsxBiI4B1sO63QxFCbBdCLLZfNwBYBWAYiug+hMUh+22Z/SdQRPcAAERUA+BiAH9SiovqHgIomvsgop6wHLu7AUAI0SKE2I9OcA9d2fgPA7BVeV9nlxUTg4QQ2wHLsAIYaJd3+nsjolEAToDlORfVfdhyyRIAuwDMF0IU3T0A+DWAbwFIK2XFdg+A1fA+R0SLiGiOXVZM9zEGQD2Ae2wJ7k9EVI1OcA9d2fiTpqyrxLV26nsjou4AHgHwFSHEwaBdNWUdfh9CiJQQYgqAGgDTiWhywO6d7h6I6BIAu4QQi6Ieoinr8M/B5nQhxIkA3g9gLhGdFbBvZ7yPBCw5904hxAkAGmHJPCba7R66svGvAzBceV8DYFsH1SVXdhLREACw/++yyzvtvRFRGSzD/4AQ4lG7uOjuAwDs7vnLAGaiuO7hdACXEdEmWHLnuUT0VxTXPQAAhBDb7P+7APwTlgRSTPdRB6DO7j0CwMOwGoMOv4eubPzfAjCWiEYTUTmA2QCe6OA6ZcsTAK62X18N4HGlfDYRVRDRaABjASzsgPq5ICKCpW2uEkL8UtlUNPdBRAOIqLf9ugrA+QBWo4juQQhxvRCiRggxCtb3/kUhxFUoonsAACKqJqIe8jWACwGsQBHdhxBiB4CtRDTeLjoPwDvoDPfQ0SPhbfkH4CJYESfrAdzY0fUJqevfAGwH0Aqr9f80gH4AXgCw1v7fV9n/Rvu+1gB4f0fX367TGbC6qMsALLH/Liqm+wBwHIC37XtYAeAmu7xo7sFzP2cjE+1TVPcASy9fav+tlL/hIryPKQBq7e/UYwD6dIZ74PQODMMwJUhXln0YhmEYA2z8GYZhShA2/gzDMCUIG3+GYZgShI0/wzBMCcLGn2EYpgRh488wDFOC/D+WyX7mrAJzOwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > 250:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
